Backend: Major code refactoring

Signed-off-by: Michael Mayer <michael@liquidbytes.net>
This commit is contained in:
Michael Mayer
2020-01-02 05:03:47 +01:00
parent 32fdb72ac9
commit 057204d379
28 changed files with 311 additions and 313 deletions

View File

@@ -16,18 +16,18 @@ import (
"github.com/photoprism/photoprism/internal/photoprism"
)
var importer *photoprism.Importer
var imp *photoprism.Import
func initImporter(conf *config.Config) {
if importer != nil {
func initImport(conf *config.Config) {
if imp != nil {
return
}
initIndexer(conf)
initIndex(conf)
converter := photoprism.NewConverter(conf)
convert := photoprism.NewConvert(conf)
importer = photoprism.NewImporter(conf, indexer, converter)
imp = photoprism.NewImport(conf, ind, convert)
}
// POST /api/v1/import*
@@ -55,9 +55,9 @@ func StartImport(router *gin.RouterGroup, conf *config.Config) {
event.Info(fmt.Sprintf("importing photos from \"%s\"", filepath.Base(path)))
initImporter(conf)
initImport(conf)
importer.Start(path)
imp.Start(path)
if subPath != "" && util.DirectoryIsEmpty(path) {
if err := os.Remove(path); err != nil {
@@ -71,7 +71,7 @@ func StartImport(router *gin.RouterGroup, conf *config.Config) {
event.Success(fmt.Sprintf("import completed in %d s", elapsed))
event.Publish("import.completed", event.Data{"path": path, "seconds": elapsed})
event.Publish("index.completed", event.Data{"path": path, "seconds": elapsed})
event.Publish("ind.completed", event.Data{"path": path, "seconds": elapsed})
event.Publish("config.updated", event.Data(conf.ClientConfig()))
c.JSON(http.StatusOK, gin.H{"message": fmt.Sprintf("import completed in %d s", elapsed)})
@@ -86,9 +86,9 @@ func CancelImport(router *gin.RouterGroup, conf *config.Config) {
return
}
initImporter(conf)
initImport(conf)
importer.Cancel()
imp.Cancel()
c.JSON(http.StatusOK, gin.H{"message": "import canceled"})
})

View File

@@ -15,32 +15,32 @@ import (
"github.com/photoprism/photoprism/internal/util"
)
var indexer *photoprism.Indexer
var nsfwDetector *nsfw.Detector
var ind *photoprism.Index
var nd *nsfw.Detector
func initIndexer(conf *config.Config) {
if indexer != nil {
func initIndex(conf *config.Config) {
if ind != nil {
return
}
initNsfwDetector(conf)
tensorFlow := photoprism.NewTensorFlow(conf)
tf := photoprism.NewTensorFlow(conf)
indexer = photoprism.NewIndexer(conf, tensorFlow, nsfwDetector)
ind = photoprism.NewIndex(conf, tf, nd)
}
func initNsfwDetector(conf *config.Config) {
if nsfwDetector != nil {
if nd != nil {
return
}
nsfwDetector = nsfw.NewDetector(conf.NSFWModelPath())
nd = nsfw.NewDetector(conf.NSFWModelPath())
}
// POST /api/v1/index
// POST /api/v1/ind
func StartIndexing(router *gin.RouterGroup, conf *config.Config) {
router.POST("/index", func(c *gin.Context) {
router.POST("/ind", func(c *gin.Context) {
if Unauthorized(c, conf) {
c.AbortWithStatusJSON(http.StatusUnauthorized, ErrUnauthorized)
return
@@ -48,7 +48,7 @@ func StartIndexing(router *gin.RouterGroup, conf *config.Config) {
start := time.Now()
var f form.IndexerOptions
var f form.IndexOptions
if err := c.BindJSON(&f); err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": util.UcFirst(err.Error())})
@@ -60,8 +60,8 @@ func StartIndexing(router *gin.RouterGroup, conf *config.Config) {
event.Info(fmt.Sprintf("indexing photos in \"%s\"", filepath.Base(path)))
if f.ConvertRaw && !conf.ReadOnly() {
converter := photoprism.NewConverter(conf)
converter.ConvertAll(conf.OriginalsPath())
convert := photoprism.NewConvert(conf)
convert.Path(conf.OriginalsPath())
}
if f.CreateThumbs {
@@ -70,35 +70,35 @@ func StartIndexing(router *gin.RouterGroup, conf *config.Config) {
}
}
initIndexer(conf)
initIndex(conf)
if f.SkipUnchanged {
indexer.Start(photoprism.IndexerOptionsNone())
ind.Start(photoprism.IndexOptionsNone())
} else {
indexer.Start(photoprism.IndexerOptionsAll())
ind.Start(photoprism.IndexOptionsAll())
}
elapsed := int(time.Since(start).Seconds())
event.Success(fmt.Sprintf("indexing completed in %d s", elapsed))
event.Publish("index.completed", event.Data{"path": path, "seconds": elapsed})
event.Publish("ind.completed", event.Data{"path": path, "seconds": elapsed})
event.Publish("config.updated", event.Data(conf.ClientConfig()))
c.JSON(http.StatusOK, gin.H{"message": fmt.Sprintf("indexing completed in %d s", elapsed)})
})
}
// DELETE /api/v1/index
// DELETE /api/v1/ind
func CancelIndexing(router *gin.RouterGroup, conf *config.Config) {
router.DELETE("/index", func(c *gin.Context) {
router.DELETE("/ind", func(c *gin.Context) {
if Unauthorized(c, conf) {
c.AbortWithStatusJSON(http.StatusUnauthorized, ErrUnauthorized)
return
}
initIndexer(conf)
initIndex(conf)
indexer.Cancel()
ind.Cancel()
c.JSON(http.StatusOK, gin.H{"message": "indexing canceled"})
})

View File

@@ -67,7 +67,7 @@ func Upload(router *gin.RouterGroup, conf *config.Config) {
containsNSFW := false
for _, filename := range uploads {
labels, err := nsfwDetector.LabelsFromFile(filename)
labels, err := nd.LabelsFromFile(filename)
if err != nil {
log.Debug(err)

View File

@@ -30,7 +30,7 @@ func wsReader(ws *websocket.Conn) {
func wsWriter(ws *websocket.Conn, conf *config.Config) {
pingTicker := time.NewTicker(10 * time.Second)
s := event.Subscribe("log.*", "notify.*", "index.*", "upload.*", "import.*", "config.*", "count.*")
s := event.Subscribe("log.*", "notify.*", "ind.*", "upload.*", "import.*", "config.*", "count.*")
defer func() {
pingTicker.Stop()

View File

@@ -30,9 +30,9 @@ func convertAction(ctx *cli.Context) error {
log.Infof("converting RAW images in %s to JPEG", conf.OriginalsPath())
converter := photoprism.NewConverter(conf)
convert := photoprism.NewConvert(conf)
converter.ConvertAll(conf.OriginalsPath())
convert.Path(conf.OriginalsPath())
elapsed := time.Since(start)

View File

@@ -43,13 +43,13 @@ func importAction(ctx *cli.Context) error {
tensorFlow := photoprism.NewTensorFlow(conf)
nsfwDetector := nsfw.NewDetector(conf.NSFWModelPath())
indexer := photoprism.NewIndexer(conf, tensorFlow, nsfwDetector)
ind := photoprism.NewIndex(conf, tensorFlow, nsfwDetector)
converter := photoprism.NewConverter(conf)
convert := photoprism.NewConvert(conf)
importer := photoprism.NewImporter(conf, indexer, converter)
imp := photoprism.NewImport(conf, ind, convert)
importer.Start(conf.ImportPath())
imp.Start(conf.ImportPath())
elapsed := time.Since(start)

View File

@@ -39,13 +39,13 @@ func indexAction(ctx *cli.Context) error {
log.Infof("read-only mode enabled")
}
tensorFlow := photoprism.NewTensorFlow(conf)
nsfwDetector := nsfw.NewDetector(conf.NSFWModelPath())
tf := photoprism.NewTensorFlow(conf)
nd := nsfw.NewDetector(conf.NSFWModelPath())
indexer := photoprism.NewIndexer(conf, tensorFlow, nsfwDetector)
ind := photoprism.NewIndex(conf, tf, nd)
options := photoprism.IndexerOptionsAll()
files := indexer.Start(options)
opt := photoprism.IndexOptionsAll()
files := ind.Start(opt)
elapsed := time.Since(start)

View File

@@ -1,6 +1,6 @@
package form
type IndexerOptions struct {
type IndexOptions struct {
SkipUnchanged bool `json:"skipUnchanged"`
CreateThumbs bool `json:"createThumbs"`
ConvertRaw bool `json:"convertRaw"`

View File

@@ -9,7 +9,7 @@ import (
"github.com/photoprism/photoprism/internal/colors"
)
// Colors returns color information for a media file.
// Colors returns the ColorPerception of an image (only JPEG supported).
func (m *MediaFile) Colors(thumbPath string) (perception colors.ColorPerception, err error) {
if !m.IsJpeg() {
return perception, errors.New("no color information: not a JPEG file")

View File

@@ -10,20 +10,18 @@ import (
"github.com/photoprism/photoprism/internal/event"
)
// Converter wraps a darktable cli binary.
type Converter struct {
// Convert represents a converter that can convert RAW/HEIF images to JPEG.
type Convert struct {
conf *config.Config
}
// NewConverter returns a new converter by setting the darktable
// cli binary location.
func NewConverter(conf *config.Config) *Converter {
return &Converter{conf: conf}
// NewConvert returns a new converter and expects the config as argument.
func NewConvert(conf *config.Config) *Convert {
return &Convert{conf: conf}
}
// ConvertAll converts all the files given a path to JPEG. This function
// ignores error during this process.
func (c *Converter) ConvertAll(path string) {
// Path converts all files in a directory to JPEG if possible.
func (c *Convert) Path(path string) {
err := filepath.Walk(path, func(filename string, fileInfo os.FileInfo, err error) error {
if err != nil {
@@ -41,7 +39,7 @@ func (c *Converter) ConvertAll(path string) {
return nil
}
if _, err := c.ConvertToJpeg(mediaFile); err != nil {
if _, err := c.ToJpeg(mediaFile); err != nil {
log.Warnf("file could not be converted to JPEG: \"%s\"", filename)
}
@@ -53,7 +51,8 @@ func (c *Converter) ConvertAll(path string) {
}
}
func (c *Converter) ConvertCommand(image *MediaFile, jpegFilename string, xmpFilename string) (result *exec.Cmd, err error) {
// ConvertCommand returns the command for converting files to JPEG, depending on the format.
func (c *Convert) ConvertCommand(image *MediaFile, jpegFilename string, xmpFilename string) (result *exec.Cmd, err error) {
if image.IsRaw() {
if c.conf.SipsBin() != "" {
result = exec.Command(c.conf.SipsBin(), "-s format jpeg", image.filename, "--out "+jpegFilename)
@@ -73,8 +72,8 @@ func (c *Converter) ConvertCommand(image *MediaFile, jpegFilename string, xmpFil
return result, nil
}
// ConvertToJpeg converts a single image the JPEG format.
func (c *Converter) ConvertToJpeg(image *MediaFile) (*MediaFile, error) {
// ToJpeg converts a single image file to JPEG if possible.
func (c *Convert) ToJpeg(image *MediaFile) (*MediaFile, error) {
if !image.Exists() {
return nil, fmt.Errorf("can not convert to jpeg, file does not exist: %s", image.Filename())
}

View File

@@ -9,15 +9,15 @@ import (
"github.com/stretchr/testify/assert"
)
func TestNewConverter(t *testing.T) {
func TestNewConvert(t *testing.T) {
conf := config.TestConfig()
converter := NewConverter(conf)
convert := NewConvert(conf)
assert.IsType(t, &Converter{}, converter)
assert.IsType(t, &Convert{}, convert)
}
func TestConverter_ConvertToJpeg(t *testing.T) {
func TestConvert_ToJpeg(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
@@ -26,21 +26,21 @@ func TestConverter_ConvertToJpeg(t *testing.T) {
conf.InitializeTestData(t)
converter := NewConverter(conf)
convert := NewConvert(conf)
jpegFilename := conf.ImportPath() + "/fern_green.jpg"
assert.Truef(t, util.Exists(jpegFilename), "file does not exist: %s", jpegFilename)
t.Logf("Testing RAW to JPEG converter with %s", jpegFilename)
t.Logf("Testing RAW to JPEG convert with %s", jpegFilename)
jpegMediaFile, err := NewMediaFile(jpegFilename)
assert.Nil(t, err)
imageJpeg, err := converter.ConvertToJpeg(jpegMediaFile)
imageJpeg, err := convert.ToJpeg(jpegMediaFile)
assert.Empty(t, err, "ConvertToJpeg() failed")
assert.Empty(t, err, "ToJpeg() failed")
infoJpeg, err := imageJpeg.Exif()
@@ -58,13 +58,13 @@ func TestConverter_ConvertToJpeg(t *testing.T) {
rawFilename := conf.ImportPath() + "/raw/IMG_2567.CR2"
t.Logf("Testing RAW to JPEG converter with %s", rawFilename)
t.Logf("Testing RAW to JPEG convert with %s", rawFilename)
rawMediaFile, err := NewMediaFile(rawFilename)
assert.Nil(t, err)
imageRaw, _ := converter.ConvertToJpeg(rawMediaFile)
imageRaw, _ := convert.ToJpeg(rawMediaFile)
assert.True(t, util.Exists(conf.ImportPath()+"/raw/IMG_2567.jpg"), "Jpeg file was not found - is Darktable installed?")
@@ -77,7 +77,7 @@ func TestConverter_ConvertToJpeg(t *testing.T) {
assert.Equal(t, "Canon EOS 6D", infoRaw.CameraModel)
}
func TestConverter_ConvertAll(t *testing.T) {
func TestConvert_Path(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
@@ -86,9 +86,9 @@ func TestConverter_ConvertAll(t *testing.T) {
conf.InitializeTestData(t)
converter := NewConverter(conf)
convert := NewConvert(conf)
converter.ConvertAll(conf.ImportPath())
convert.Path(conf.ImportPath())
jpegFilename := conf.ImportPath() + "/raw/canon_eos_6d.jpg"
@@ -112,7 +112,7 @@ func TestConverter_ConvertAll(t *testing.T) {
os.Remove(existingJpegFilename)
converter.ConvertAll(conf.ImportPath())
convert.Path(conf.ImportPath())
newHash := util.Hash(existingJpegFilename)

View File

@@ -12,7 +12,7 @@ import (
"gopkg.in/ugjka/go-tz.v2/tz"
)
// Exif returns information about a single image.
// Exif represents MediaFile metadata.
type Exif struct {
UUID string
TakenAt time.Time

View File

@@ -137,9 +137,9 @@ func TestMediaFile_Exif_HEIF(t *testing.T) {
assert.Nil(t, err)
converter := NewConverter(conf)
convert := NewConvert(conf)
jpeg, err := converter.ConvertToJpeg(img)
jpeg, err := convert.ToJpeg(img)
assert.Nil(t, err)

View File

@@ -16,22 +16,22 @@ import (
"github.com/photoprism/photoprism/internal/util"
)
// Importer is responsible for importing new files to originals.
type Importer struct {
// Import represents an importer that can copy/move MediaFiles to the originals directory.
type Import struct {
conf *config.Config
indexer *Indexer
converter *Converter
index *Index
convert *Convert
removeDotFiles bool
removeExistingFiles bool
removeEmptyDirectories bool
}
// NewImporter returns a new importer.
func NewImporter(conf *config.Config, indexer *Indexer, converter *Converter) *Importer {
instance := &Importer{
// NewImport returns a new importer and expects its dependencies as arguments.
func NewImport(conf *config.Config, index *Index, convert *Convert) *Import {
instance := &Import{
conf: conf,
indexer: indexer,
converter: converter,
index: index,
convert: convert,
removeDotFiles: true,
removeExistingFiles: true,
removeEmptyDirectories: true,
@@ -40,19 +40,18 @@ func NewImporter(conf *config.Config, indexer *Indexer, converter *Converter) *I
return instance
}
func (imp *Importer) originalsPath() string {
func (imp *Import) originalsPath() string {
return imp.conf.OriginalsPath()
}
// Start imports all the photos from a given directory path.
// This function ignores errors.
func (imp *Importer) Start(importPath string) {
// Start imports MediaFiles from a directory and converts/indexes them as needed.
func (imp *Import) Start(importPath string) {
var directories []string
done := make(map[string]bool)
ind := imp.indexer
ind := imp.index
if ind.running {
event.Error("indexer already running")
event.Error("index already running")
return
}
@@ -77,12 +76,12 @@ func (imp *Importer) Start(importPath string) {
wg.Add(numWorkers)
for i := 0; i < numWorkers; i++ {
go func() {
importerWorker(jobs) // HLc
importWorker(jobs) // HLc
wg.Done()
}()
}
options := IndexerOptionsAll()
options := IndexOptionsAll()
err := filepath.Walk(importPath, func(filename string, fileInfo os.FileInfo, err error) error {
defer func() {
@@ -134,10 +133,10 @@ func (imp *Importer) Start(importPath string) {
}
jobs <- ImportJob{
related: related,
options: options,
importPath: importPath,
imp: imp,
related: related,
opt: options,
path: importPath,
imp: imp,
}
return nil
@@ -169,12 +168,12 @@ func (imp *Importer) Start(importPath string) {
}
// Cancel stops the current import operation.
func (imp *Importer) Cancel() {
imp.indexer.Cancel()
func (imp *Import) Cancel() {
imp.index.Cancel()
}
// DestinationFilename get the destination of a media file.
func (imp *Importer) DestinationFilename(mainFile *MediaFile, mediaFile *MediaFile) (string, error) {
// DestinationFilename returns the destination filename of a MediaFile to be imported.
func (imp *Import) DestinationFilename(mainFile *MediaFile, mediaFile *MediaFile) (string, error) {
fileName := mainFile.CanonicalName()
fileExtension := mediaFile.Extension()
dateCreated := mainFile.DateCreated()

View File

@@ -0,0 +1,70 @@
package photoprism
import (
"testing"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/nsfw"
"github.com/stretchr/testify/assert"
)
func TestNewImport(t *testing.T) {
conf := config.TestConfig()
tf := NewTensorFlow(conf)
nd := nsfw.NewDetector(conf.NSFWModelPath())
ind := NewIndex(conf, tf, nd)
convert := NewConvert(conf)
imp := NewImport(conf, ind, convert)
assert.IsType(t, &Import{}, imp)
}
func TestImport_DestinationFilename(t *testing.T) {
conf := config.TestConfig()
conf.InitializeTestData(t)
tf := NewTensorFlow(conf)
nd := nsfw.NewDetector(conf.NSFWModelPath())
ind := NewIndex(conf, tf, nd)
convert := NewConvert(conf)
imp := NewImport(conf, ind, convert)
rawFile, err := NewMediaFile(conf.ImportPath() + "/raw/IMG_2567.CR2")
assert.Nil(t, err)
filename, _ := imp.DestinationFilename(rawFile, rawFile)
// TODO: Check for errors!
assert.Equal(t, conf.OriginalsPath()+"/2019/07/20190705_153230_6E16EB388AD2.cr2", filename)
}
func TestImport_Start(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
conf := config.TestConfig()
conf.InitializeTestData(t)
tf := NewTensorFlow(conf)
nd := nsfw.NewDetector(conf.NSFWModelPath())
ind := NewIndex(conf, tf, nd)
convert := NewConvert(conf)
imp := NewImport(conf, ind, convert)
imp.Start(conf.ImportPath())
}

View File

@@ -9,46 +9,46 @@ import (
)
type ImportJob struct {
related RelatedFiles
options IndexerOptions
importPath string
imp *Importer
related RelatedFiles
opt IndexOptions
path string
imp *Import
}
func importerWorker(jobs <-chan ImportJob) {
func importWorker(jobs <-chan ImportJob) {
for job := range jobs {
var destinationMainFilename string
related := job.related
imp := job.imp
options := job.options
importPath := job.importPath
opt := job.opt
importPath := job.path
event.Publish("import.file", event.Data{
"fileName": related.main.Filename(),
"baseName": filepath.Base(related.main.Filename()),
})
for _, relatedMediaFile := range related.files {
relativeFilename := relatedMediaFile.RelativeFilename(importPath)
for _, f := range related.files {
relativeFilename := f.RelativeFilename(importPath)
if destinationFilename, err := imp.DestinationFilename(related.main, relatedMediaFile); err == nil {
if destinationFilename, err := imp.DestinationFilename(related.main, f); err == nil {
if err := os.MkdirAll(path.Dir(destinationFilename), os.ModePerm); err != nil {
log.Errorf("import: could not create directories (%s)", err.Error())
}
if related.main.HasSameFilename(relatedMediaFile) {
if related.main.HasSameFilename(f) {
destinationMainFilename = destinationFilename
log.Infof("import: moving main %s file \"%s\" to \"%s\"", relatedMediaFile.Type(), relativeFilename, destinationFilename)
log.Infof("import: moving main %s file \"%s\" to \"%s\"", f.Type(), relativeFilename, destinationFilename)
} else {
log.Infof("import: moving related %s file \"%s\" to \"%s\"", relatedMediaFile.Type(), relativeFilename, destinationFilename)
log.Infof("import: moving related %s file \"%s\" to \"%s\"", f.Type(), relativeFilename, destinationFilename)
}
if err := relatedMediaFile.Move(destinationFilename); err != nil {
if err := f.Move(destinationFilename); err != nil {
log.Errorf("import: could not move file to \"%s\" (%s)", destinationMainFilename, err.Error())
}
} else if imp.removeExistingFiles {
if err := relatedMediaFile.Remove(); err != nil {
log.Errorf("import: could not delete file \"%s\" (%s)", relatedMediaFile.Filename(), err.Error())
if err := f.Remove(); err != nil {
log.Errorf("import: could not delete file \"%s\" (%s)", f.Filename(), err.Error())
} else {
log.Infof("import: deleted \"%s\" (already exists)", relativeFilename)
}
@@ -65,12 +65,12 @@ func importerWorker(jobs <-chan ImportJob) {
}
if importedMainFile.IsRaw() {
if _, err := imp.converter.ConvertToJpeg(importedMainFile); err != nil {
if _, err := imp.convert.ToJpeg(importedMainFile); err != nil {
log.Errorf("import: could not create jpeg from raw (%s)", err)
}
}
if importedMainFile.IsHEIF() {
if _, err := imp.converter.ConvertToJpeg(importedMainFile); err != nil {
if _, err := imp.convert.ToJpeg(importedMainFile); err != nil {
log.Errorf("import: could not create jpeg from heif (%s)", err)
}
}
@@ -83,22 +83,22 @@ func importerWorker(jobs <-chan ImportJob) {
}
}
indexed := make(map[string]bool)
ind := imp.indexer
mainIndexResult := ind.indexMediaFile(related.main, options)
indexed[related.main.Filename()] = true
done := make(map[string]bool)
ind := imp.index
res := ind.MediaFile(related.main, opt)
done[related.main.Filename()] = true
log.Infof("import: indexed %s main %s file \"%s\"", mainIndexResult, related.main.Type(), related.main.RelativeFilename(ind.originalsPath()))
log.Infof("import: %s main %s file \"%s\"", res, related.main.Type(), related.main.RelativeFilename(ind.originalsPath()))
for _, relatedMediaFile := range related.files {
if indexed[relatedMediaFile.Filename()] {
for _, f := range related.files {
if done[f.Filename()] {
continue
}
indexResult := ind.indexMediaFile(relatedMediaFile, options)
indexed[relatedMediaFile.Filename()] = true
res := ind.MediaFile(f, opt)
done[f.Filename()] = true
log.Infof("import: indexed %s related %s file \"%s\"", indexResult, relatedMediaFile.Type(), relatedMediaFile.RelativeFilename(ind.originalsPath()))
log.Infof("import: %s related %s file \"%s\"", res, f.Type(), f.RelativeFilename(ind.originalsPath()))
}
}
}

View File

@@ -1,70 +0,0 @@
package photoprism
import (
"testing"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/nsfw"
"github.com/stretchr/testify/assert"
)
func TestNewImporter(t *testing.T) {
conf := config.TestConfig()
tensorFlow := NewTensorFlow(conf)
nsfwDetector := nsfw.NewDetector(conf.NSFWModelPath())
indexer := NewIndexer(conf, tensorFlow, nsfwDetector)
converter := NewConverter(conf)
importer := NewImporter(conf, indexer, converter)
assert.IsType(t, &Importer{}, importer)
}
func TestImporter_DestinationFilename(t *testing.T) {
conf := config.TestConfig()
conf.InitializeTestData(t)
tensorFlow := NewTensorFlow(conf)
nsfwDetector := nsfw.NewDetector(conf.NSFWModelPath())
indexer := NewIndexer(conf, tensorFlow, nsfwDetector)
converter := NewConverter(conf)
importer := NewImporter(conf, indexer, converter)
rawFile, err := NewMediaFile(conf.ImportPath() + "/raw/IMG_2567.CR2")
assert.Nil(t, err)
filename, _ := importer.DestinationFilename(rawFile, rawFile)
// TODO: Check for errors!
assert.Equal(t, conf.OriginalsPath()+"/2019/07/20190705_153230_6E16EB388AD2.cr2", filename)
}
func TestImporter_Start(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
conf := config.TestConfig()
conf.InitializeTestData(t)
tensorFlow := NewTensorFlow(conf)
nsfwDetector := nsfw.NewDetector(conf.NSFWModelPath())
indexer := NewIndexer(conf, tensorFlow, nsfwDetector)
converter := NewConverter(conf)
importer := NewImporter(conf, indexer, converter)
importer.Start(conf.ImportPath())
}

View File

@@ -13,8 +13,8 @@ import (
"github.com/photoprism/photoprism/internal/nsfw"
)
// Indexer defines an indexer with originals path tensorflow and a db.
type Indexer struct {
// Index represents an indexer that indexes files in the originals directory.
type Index struct {
conf *config.Config
tensorFlow *TensorFlow
nsfwDetector *nsfw.Detector
@@ -23,9 +23,9 @@ type Indexer struct {
canceled bool
}
// NewIndexer returns a new indexer.
func NewIndexer(conf *config.Config, tensorFlow *TensorFlow, nsfwDetector *nsfw.Detector) *Indexer {
i := &Indexer{
// NewIndex returns a new indexer and expects its dependencies as arguments.
func NewIndex(conf *config.Config, tensorFlow *TensorFlow, nsfwDetector *nsfw.Detector) *Index {
i := &Index{
conf: conf,
tensorFlow: tensorFlow,
nsfwDetector: nsfwDetector,
@@ -35,25 +35,25 @@ func NewIndexer(conf *config.Config, tensorFlow *TensorFlow, nsfwDetector *nsfw.
return i
}
func (ind *Indexer) originalsPath() string {
func (ind *Index) originalsPath() string {
return ind.conf.OriginalsPath()
}
func (ind *Indexer) thumbnailsPath() string {
func (ind *Index) thumbnailsPath() string {
return ind.conf.ThumbnailsPath()
}
// Cancel stops the current indexing operation.
func (ind *Indexer) Cancel() {
func (ind *Index) Cancel() {
ind.canceled = true
}
// Start will index mediafiles in the originals directory.
func (ind *Indexer) Start(options IndexerOptions) map[string]bool {
// Start will index MediaFiles in the originals directory.
func (ind *Index) Start(options IndexOptions) map[string]bool {
done := make(map[string]bool)
if ind.running {
event.Error("indexer already running")
event.Error("index already running")
return done
}
@@ -79,7 +79,7 @@ func (ind *Indexer) Start(options IndexerOptions) map[string]bool {
wg.Add(numWorkers)
for i := 0; i < numWorkers; i++ {
go func() {
indexerWorker(jobs) // HLc
indexWorker(jobs) // HLc
wg.Done()
}()
}
@@ -123,7 +123,7 @@ func (ind *Indexer) Start(options IndexerOptions) map[string]bool {
jobs <- IndexJob{
related: related,
options: options,
opt: options,
ind: ind,
}

View File

@@ -23,7 +23,7 @@ const (
type IndexResult string
func (ind *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
func (ind *Index) MediaFile(m *MediaFile, o IndexOptions) IndexResult {
start := time.Now()
var photo entity.Photo
@@ -247,7 +247,7 @@ func (ind *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
}
// classifyImage returns all matching labels for a media file.
func (ind *Indexer) classifyImage(jpeg *MediaFile) (results Labels, isNSFW bool) {
func (ind *Index) classifyImage(jpeg *MediaFile) (results Labels, isNSFW bool) {
start := time.Now()
var thumbs []string
@@ -323,7 +323,7 @@ func (ind *Indexer) classifyImage(jpeg *MediaFile) (results Labels, isNSFW bool)
return results, isNSFW
}
func (ind *Indexer) addLabels(photoId uint, labels Labels) {
func (ind *Index) addLabels(photoId uint, labels Labels) {
for _, label := range labels {
lm := entity.NewLabel(label.Name, label.Priority).FirstOrCreate(ind.db)
@@ -361,7 +361,7 @@ func (ind *Indexer) addLabels(photoId uint, labels Labels) {
}
}
func (ind *Indexer) indexLocation(mediaFile *MediaFile, photo *entity.Photo, labels Labels, fileChanged bool, o IndexerOptions) ([]string, Labels) {
func (ind *Index) indexLocation(mediaFile *MediaFile, photo *entity.Photo, labels Labels, fileChanged bool, o IndexOptions) ([]string, Labels) {
var keywords []string
if location, err := mediaFile.Location(); err == nil {
@@ -442,7 +442,7 @@ func (ind *Indexer) indexLocation(mediaFile *MediaFile, photo *entity.Photo, lab
return keywords, labels
}
func (ind *Indexer) estimateLocation(photo *entity.Photo) {
func (ind *Index) estimateLocation(photo *entity.Photo) {
var recentPhoto entity.Photo
if result := ind.db.Unscoped().Order(gorm.Expr("ABS(DATEDIFF(taken_at, ?)) ASC", photo.TakenAt)).Preload("Place").First(&recentPhoto); result.Error == nil {

View File

@@ -4,7 +4,7 @@ import (
"reflect"
)
type IndexerOptions struct {
type IndexOptions struct {
UpdateDate bool
UpdateColors bool
UpdateSize bool
@@ -17,7 +17,7 @@ type IndexerOptions struct {
UpdateExif bool
}
func (o *IndexerOptions) UpdateAny() bool {
func (o *IndexOptions) UpdateAny() bool {
v := reflect.ValueOf(o).Elem()
for i := 0; i < v.NumField(); i++ {
@@ -29,13 +29,13 @@ func (o *IndexerOptions) UpdateAny() bool {
return false
}
func (o *IndexerOptions) SkipUnchanged() bool {
func (o *IndexOptions) SkipUnchanged() bool {
return !o.UpdateAny()
}
// IndexerOptionsAll returns new indexer options with all options set to true.
func IndexerOptionsAll() IndexerOptions {
instance := IndexerOptions{
// IndexOptionsAll returns new index options with all options set to true.
func IndexOptionsAll() IndexOptions {
result := IndexOptions{
UpdateDate: true,
UpdateColors: true,
UpdateSize: true,
@@ -48,12 +48,12 @@ func IndexerOptionsAll() IndexerOptions {
UpdateExif: true,
}
return instance
return result
}
// IndexerOptionsNone returns new indexer options with all options set to false.
func IndexerOptionsNone() IndexerOptions {
instance := IndexerOptions{}
// IndexOptionsNone returns new index options with all options set to false.
func IndexOptionsNone() IndexOptions {
result := IndexOptions{}
return instance
return result
}

View File

@@ -0,0 +1,33 @@
package photoprism
import (
"testing"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/nsfw"
)
func TestIndex_Start(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
conf := config.TestConfig()
conf.InitializeTestData(t)
tf := NewTensorFlow(conf)
nd := nsfw.NewDetector(conf.NSFWModelPath())
ind := NewIndex(conf, tf, nd)
convert := NewConvert(conf)
imp := NewImport(conf, ind, convert)
imp.Start(conf.ImportPath())
opt := IndexOptionsAll()
ind.Start(opt)
}

View File

@@ -0,0 +1,33 @@
package photoprism
type IndexJob struct {
related RelatedFiles
opt IndexOptions
ind *Index
}
func indexWorker(jobs <-chan IndexJob) {
for job := range jobs {
done := make(map[string]bool)
related := job.related
opt := job.opt
ind := job.ind
res := ind.MediaFile(related.main, opt)
done[related.main.Filename()] = true
log.Infof("index: %s main %s file \"%s\"", res, related.main.Type(), related.main.RelativeFilename(ind.originalsPath()))
for _, f := range related.files {
if done[f.Filename()] {
continue
}
res := ind.MediaFile(f, opt)
done[f.Filename()] = true
log.Infof("index: %s related %s file \"%s\"", res, f.Type(), f.RelativeFilename(ind.originalsPath()))
}
}
}

View File

@@ -1,33 +0,0 @@
package photoprism
import (
"testing"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/nsfw"
)
func TestIndexer_Start(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
conf := config.TestConfig()
conf.InitializeTestData(t)
tensorFlow := NewTensorFlow(conf)
nsfwDetector := nsfw.NewDetector(conf.NSFWModelPath())
indexer := NewIndexer(conf, tensorFlow, nsfwDetector)
converter := NewConverter(conf)
importer := NewImporter(conf, indexer, converter)
importer.Start(conf.ImportPath())
options := IndexerOptionsAll()
indexer.Start(options)
}

View File

@@ -1,33 +0,0 @@
package photoprism
type IndexJob struct {
related RelatedFiles
options IndexerOptions
ind *Indexer
}
func indexerWorker(jobs <-chan IndexJob) {
for job := range jobs {
indexed := make(map[string]bool)
related := job.related
options := job.options
ind := job.ind
mainIndexResult := ind.indexMediaFile(related.main, options)
indexed[related.main.Filename()] = true
log.Infof("index: %s main %s file \"%s\"", mainIndexResult, related.main.Type(), related.main.RelativeFilename(ind.originalsPath()))
for _, relatedMediaFile := range related.files {
if indexed[relatedMediaFile.Filename()] {
continue
}
indexResult := ind.indexMediaFile(relatedMediaFile, options)
indexed[relatedMediaFile.Filename()] = true
log.Infof("index: %s related %s file \"%s\"", indexResult, relatedMediaFile.Type(), relatedMediaFile.RelativeFilename(ind.originalsPath()))
}
}
}

View File

@@ -16,7 +16,7 @@ import (
"github.com/photoprism/photoprism/internal/util"
)
// MediaFile represents a single file.
// MediaFile represents a single photo, video or sidecar file.
type MediaFile struct {
filename string
dateCreated time.Time
@@ -222,13 +222,13 @@ func (m *MediaFile) CanonicalNameFromFile() string {
return basename
}
// CanonicalNameFromFileWithDirectory gets the canonical name for a mediafile
// CanonicalNameFromFileWithDirectory gets the canonical name for a MediaFile
// including the directory.
func (m *MediaFile) CanonicalNameFromFileWithDirectory() string {
return m.Directory() + string(os.PathSeparator) + m.CanonicalNameFromFile()
}
// Hash return a sha1 hash of a mediafile based on the filename.
// Hash return a sha1 hash of a MediaFile based on the filename.
func (m *MediaFile) Hash() string {
if len(m.hash) == 0 {
m.hash = util.Hash(m.Filename())
@@ -434,7 +434,7 @@ func (m *MediaFile) Move(newFilename string) error {
return nil
}
// Copy a mediafile to another file by destinationFilename.
// Copy a MediaFile to another file by destinationFilename.
func (m *MediaFile) Copy(destinationFilename string) error {
file, err := m.openFile()
@@ -593,7 +593,7 @@ func (m *MediaFile) decodeDimensions() error {
return nil
}
// Width return the width dimension of a mediafile.
// Width return the width dimension of a MediaFile.
func (m *MediaFile) Width() int {
if !m.IsPhoto() {
return 0
@@ -608,7 +608,7 @@ func (m *MediaFile) Width() int {
return m.width
}
// Height returns the height dimension of a mediafile.
// Height returns the height dimension of a MediaFile.
func (m *MediaFile) Height() int {
if !m.IsPhoto() {
return 0
@@ -623,7 +623,7 @@ func (m *MediaFile) Height() int {
return m.height
}
// AspectRatio returns the aspect ratio of a mediafile.
// AspectRatio returns the aspect ratio of a MediaFile.
func (m *MediaFile) AspectRatio() float64 {
width := float64(m.Width())
height := float64(m.Height())
@@ -637,7 +637,7 @@ func (m *MediaFile) AspectRatio() float64 {
return aspectRatio
}
// Orientation returns the orientation of a mediafile.
// Orientation returns the orientation of a MediaFile.
func (m *MediaFile) Orientation() int {
if exif, err := m.Exif(); err == nil {
return exif.Orientation

View File

@@ -1,14 +1,14 @@
package photoprism
// MediaFiles provides a Collection for mediafiles.
// MediaFiles provides a Collection for MediaFiles.
type MediaFiles []*MediaFile
// Len returns the length of the mediafile collection.
// Len returns the length of the MediaFile collection.
func (f MediaFiles) Len() int {
return len(f)
}
// Less compares two mediafiles based on filename length.
// Less compares two MediaFiles based on filename length.
func (f MediaFiles) Less(i, j int) bool {
fileName1 := f[i].Filename()
fileName2 := f[j].Filename()

View File

@@ -66,16 +66,16 @@ func TestThumbnails_CreateThumbnailsFromOriginals(t *testing.T) {
conf.InitializeTestData(t)
tensorFlow := NewTensorFlow(conf)
nsfwDetector := nsfw.NewDetector(conf.NSFWModelPath())
tf := NewTensorFlow(conf)
nd := nsfw.NewDetector(conf.NSFWModelPath())
indexer := NewIndexer(conf, tensorFlow, nsfwDetector)
ind := NewIndex(conf, tf, nd)
converter := NewConverter(conf)
convert := NewConvert(conf)
importer := NewImporter(conf, indexer, converter)
imp := NewImport(conf, ind, convert)
importer.Start(conf.ImportPath())
imp.Start(conf.ImportPath())
err := CreateThumbnailsFromOriginals(conf.OriginalsPath(), conf.ThumbnailsPath(), true)

View File

@@ -30,7 +30,7 @@ func (s *Repo) FindFileByPhotoUUID(u string) (file entity.File, err error) {
return file, nil
}
// FindFileByID returns a mediafile given a certain ID.
// FindFileByID returns a MediaFile given a certain ID.
func (s *Repo) FindFileByID(id string) (file entity.File, err error) {
if err := s.db.Where("id = ?", id).Preload("Photo").First(&file).Error; err != nil {
return file, err