From eb8bc7b709daa2933a157265d4b8900d596f549c Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Thu, 23 Sep 2021 23:46:17 +0200 Subject: [PATCH] Backend: Improve resilience #1544 --- docker-compose.db.yml | 64 +++++++++++++++++++++ docker-compose.latest.yml | 59 +++++++++++++++++++ docker-compose.proxy.yml | 20 +++++++ docker-compose.yml | 35 +++--------- frontend/src/model/rest.js | 8 ++- internal/api/cache.go | 6 +- internal/api/subject.go | 2 +- internal/classify/gen.go | 2 +- internal/classify/label.go | 27 +-------- internal/classify/label_test.go | 49 ---------------- internal/classify/rules.go | 2 +- internal/classify/rules_test.go | 2 +- internal/classify/tensorflow.go | 2 +- internal/entity/account.go | 12 ++-- internal/entity/album.go | 16 +++--- internal/entity/camera.go | 24 ++++---- internal/entity/country.go | 12 ++-- internal/entity/details.go | 19 ++++--- internal/entity/entity.go | 13 +++-- internal/entity/face.go | 4 +- internal/entity/file.go | 17 ++++-- internal/entity/file_test.go | 30 +++++++++- internal/entity/folder.go | 16 +++--- internal/entity/label.go | 17 +++--- internal/entity/lens.go | 24 ++++---- internal/entity/link.go | 7 +-- internal/entity/marker.go | 45 ++++++++++++++- internal/entity/markers.go | 75 ++++++++++++++++++++++-- internal/entity/markers_test.go | 79 ++++++++++++++++++++++++-- internal/entity/photo.go | 10 ++-- internal/entity/photo_counts.go | 13 +++++ internal/entity/photo_title.go | 6 +- internal/entity/subject.go | 43 +++++++------- internal/entity/subject_test.go | 4 +- internal/hub/feedback.go | 2 +- internal/photoprism/index_mediafile.go | 32 +++++++---- internal/query/previews.go | 66 +++++++++++++++------ internal/search/photos.go | 12 ++-- internal/search/photos_geo.go | 6 +- internal/search/photos_test.go | 18 +++--- pkg/txt/clip.go | 26 ++++++--- pkg/txt/clip_test.go | 30 ++++++---- pkg/txt/names.go | 18 +----- pkg/txt/names_test.go | 15 ----- pkg/txt/quote.go | 5 ++ pkg/txt/quote_test.go | 12 ++++ pkg/txt/slug.go | 17 +++++- pkg/txt/slug_test.go | 15 +++++ scripts/sql/init-test-databases.sql | 4 +- 49 files changed, 703 insertions(+), 339 deletions(-) create mode 100644 docker-compose.db.yml create mode 100644 docker-compose.latest.yml create mode 100644 docker-compose.proxy.yml diff --git a/docker-compose.db.yml b/docker-compose.db.yml new file mode 100644 index 000000000..ae1284312 --- /dev/null +++ b/docker-compose.db.yml @@ -0,0 +1,64 @@ +version: '3.5' + +# Legacy databases servers for testing. +services: + mariadb-10-3: + image: mariadb:10.3 + container_name: mariadb-10-3 + command: mysqld --port=4001 --transaction-isolation=READ-COMMITTED --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=512 --innodb-rollback-on-timeout=OFF --innodb-lock-wait-timeout=50 + expose: + - "4001" + volumes: + - "./scripts/sql/init-test-databases.sql:/docker-entrypoint-initdb.d/init-test-databases.sql" + environment: + MYSQL_ROOT_PASSWORD: photoprism + MYSQL_USER: photoprism + MYSQL_PASSWORD: photoprism + MYSQL_DATABASE: photoprism + + mariadb-10-2: + image: mariadb:10.2 + container_name: mariadb-10-2 + command: mysqld --port=4001 --transaction-isolation=READ-COMMITTED --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=512 --innodb-rollback-on-timeout=OFF --innodb-lock-wait-timeout=50 + expose: + - "4001" + volumes: + - "./scripts/sql/init-test-databases.sql:/docker-entrypoint-initdb.d/init-test-databases.sql" + environment: + MYSQL_ROOT_PASSWORD: photoprism + MYSQL_USER: photoprism + MYSQL_PASSWORD: photoprism + MYSQL_DATABASE: photoprism + + mariadb-10-1: + image: mariadb:10.1 + container_name: mariadb-10-1 + command: mysqld --port=4001 --transaction-isolation=READ-COMMITTED --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=512 --innodb-rollback-on-timeout=OFF --innodb-lock-wait-timeout=50 + expose: + - "4001" + volumes: + - "./scripts/sql/init-test-databases.sql:/docker-entrypoint-initdb.d/init-test-databases.sql" + environment: + MYSQL_ROOT_PASSWORD: photoprism + MYSQL_USER: photoprism + MYSQL_PASSWORD: photoprism + MYSQL_DATABASE: photoprism + + mysql-8: + image: mysql:8 + container_name: mysql-8 + command: mysqld --port=4001 --transaction-isolation=READ-COMMITTED --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=512 --innodb-rollback-on-timeout=OFF --innodb-lock-wait-timeout=50 + expose: + - "4001" + volumes: + - "./scripts/sql/init-test-databases.sql:/docker-entrypoint-initdb.d/init-test-databases.sql" + environment: + MYSQL_ROOT_PASSWORD: photoprism + MYSQL_USER: photoprism + MYSQL_PASSWORD: photoprism + MYSQL_DATABASE: photoprism + +networks: + default: + external: + name: shared \ No newline at end of file diff --git a/docker-compose.latest.yml b/docker-compose.latest.yml new file mode 100644 index 000000000..338ca2e0e --- /dev/null +++ b/docker-compose.latest.yml @@ -0,0 +1,59 @@ +version: '3.5' + +# Latest stable version for testing. +services: + photoprism-latest: + image: photoprism/photoprism:latest + container_name: photoprism-latest + security_opt: + - seccomp:unconfined + - apparmor:unconfined + ports: + - "2344:2342" # [local port]:[container port] + environment: + UID: ${UID:-1000} + PHOTOPRISM_SITE_URL: "http://localhost:2344/" + PHOTOPRISM_SITE_TITLE: "PhotoPrism" + PHOTOPRISM_SITE_CAPTION: "Browse Your Life" + PHOTOPRISM_SITE_DESCRIPTION: "Open-Source Photo Management" + PHOTOPRISM_SITE_AUTHOR: "@photoprism_app" + PHOTOPRISM_DEBUG: "true" + PHOTOPRISM_READONLY: "false" + PHOTOPRISM_PUBLIC: "true" + PHOTOPRISM_EXPERIMENTAL: "false" + PHOTOPRISM_SERVER_MODE: "debug" + PHOTOPRISM_HTTP_HOST: "0.0.0.0" + PHOTOPRISM_HTTP_PORT: 2342 + PHOTOPRISM_HTTP_COMPRESSION: "gzip" # Improves transfer speed and bandwidth utilization (none or gzip) + PHOTOPRISM_DATABASE_DRIVER: "mysql" + PHOTOPRISM_DATABASE_SERVER: "mariadb:4001" + PHOTOPRISM_DATABASE_NAME: "latest" + PHOTOPRISM_DATABASE_USER: "root" + PHOTOPRISM_DATABASE_PASSWORD: "photoprism" + PHOTOPRISM_ADMIN_PASSWORD: "photoprism" # The initial admin password (min 4 characters) + PHOTOPRISM_DISABLE_BACKUPS: "false" # Don't backup photo and album metadata to YAML files + PHOTOPRISM_DISABLE_WEBDAV: "false" # Disables built-in WebDAV server + PHOTOPRISM_DISABLE_SETTINGS: "false" # Disables Settings in Web UI + PHOTOPRISM_DISABLE_PLACES: "false" # Disables reverse geocoding and maps + PHOTOPRISM_DISABLE_EXIFTOOL: "false" # Don't create ExifTool JSON files for improved metadata extraction + PHOTOPRISM_DISABLE_TENSORFLOW: "false" # Don't use TensorFlow for image classification + PHOTOPRISM_DETECT_NSFW: "false" # Flag photos as private that MAY be offensive (requires TensorFlow) + PHOTOPRISM_UPLOAD_NSFW: "false" # Allows uploads that may be offensive + PHOTOPRISM_DARKTABLE_PRESETS: "false" # Enables Darktable presets and disables concurrent RAW conversion + PHOTOPRISM_THUMB_FILTER: "lanczos" # Resample filter, best to worst: blackman, lanczos, cubic, linear + PHOTOPRISM_THUMB_UNCACHED: "true" # Enables on-demand thumbnail rendering (high memory and cpu usage) + PHOTOPRISM_THUMB_SIZE: 2048 # Pre-rendered thumbnail size limit (default 2048, min 720, max 7680) + # PHOTOPRISM_THUMB_SIZE: 4096 # Retina 4K, DCI 4K (requires more storage); 7680 for 8K Ultra HD + PHOTOPRISM_THUMB_SIZE_UNCACHED: 7680 # On-demand rendering size limit (default 7680, min 720, max 7680) + PHOTOPRISM_JPEG_SIZE: 7680 # Size limit for converted image files in pixels (720-30000) + PHOTOPRISM_JPEG_QUALITY: 92 # Set to 95 for high-quality thumbnails (25-100) + TF_CPP_MIN_LOG_LEVEL: 0 # Show TensorFlow log messages for development + working_dir: "/photoprism" + volumes: + - "./storage/latest:/photoprism/storage" + - "./storage/originals:/photoprism/originals" + +networks: + default: + external: + name: shared \ No newline at end of file diff --git a/docker-compose.proxy.yml b/docker-compose.proxy.yml new file mode 100644 index 000000000..dee04552c --- /dev/null +++ b/docker-compose.proxy.yml @@ -0,0 +1,20 @@ +version: '3.5' + +# Reverse proxy servers for testing. +services: + caddy: + image: caddy:2 + container_name: caddy + depends_on: + - photoprism + ports: + - "80:80" + - "443:443" + volumes: + - ./docker/development/caddy:/data/caddy/pki/authorities/local + - ./docker/development/caddy/Caddyfile:/etc/caddy/Caddyfile + +networks: + default: + external: + name: shared \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 66bf919a7..6452112e8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,7 @@ services: photoprism: build: . image: photoprism/photoprism:develop + container_name: photoprism depends_on: - mariadb - webdav-dummy @@ -65,6 +66,7 @@ services: mariadb: image: mariadb:10.5 + container_name: mariadb command: mysqld --port=4001 --transaction-isolation=READ-COMMITTED --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=512 --innodb-rollback-on-timeout=OFF --innodb-lock-wait-timeout=50 expose: - "4001" @@ -81,34 +83,11 @@ services: webdav-dummy: image: photoprism/webdav:20210602 - # Uncomment to test with MySQL 8: - # - # mysql: - # image: mysql:8 - # command: mysqld --port=4001 --transaction-isolation=READ-COMMITTED --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=512 --innodb-rollback-on-timeout=OFF --innodb-lock-wait-timeout=50 - # expose: - # - "4001" - # volumes: - # - "./scripts/sql/init-test-databases.sql:/docker-entrypoint-initdb.d/init-test-databases.sql" - # environment: - # MYSQL_ROOT_PASSWORD: photoprism - # MYSQL_USER: photoprism - # MYSQL_PASSWORD: photoprism - # MYSQL_DATABASE: photoprism - - # Uncomment to test with Caddy as reverse proxy: - # - # caddy: - # image: caddy:2 - # depends_on: - # - photoprism - # ports: - # - "80:80" - # - "443:443" - # volumes: - # - ./docker/development/caddy:/data/caddy/pki/authorities/local - # - ./docker/development/caddy/Caddyfile:/etc/caddy/Caddyfile - volumes: go-mod: driver: local + +networks: + default: + name: shared + driver: bridge diff --git a/frontend/src/model/rest.js b/frontend/src/model/rest.js index 978525cc1..60dddf72f 100644 --- a/frontend/src/model/rest.js +++ b/frontend/src/model/rest.js @@ -188,7 +188,7 @@ export class Rest extends Model { }; return Api.get(this.getCollectionResource(), options).then((resp) => { - let count = resp.data.length; + let count = resp.data ? resp.data.length : 0; let limit = 0; let offset = 0; @@ -211,8 +211,10 @@ export class Rest extends Model { resp.limit = limit; resp.offset = offset; - for (let i = 0; i < resp.data.length; i++) { - resp.models.push(new this(resp.data[i])); + if (count > 0) { + for (let i = 0; i < resp.data.length; i++) { + resp.models.push(new this(resp.data[i])); + } } return Promise.resolve(resp); diff --git a/internal/api/cache.go b/internal/api/cache.go index 191dcfcf5..d370c3b77 100644 --- a/internal/api/cache.go +++ b/internal/api/cache.go @@ -47,7 +47,7 @@ func RemoveFromFolderCache(rootName string) { cache.Delete(cacheKey) if err := query.UpdateAlbumFolderPreviews(); err != nil { - log.Errorf("failed updating folder previews: %s", err) + log.Error(err) } log.Debugf("removed %s from cache", cacheKey) @@ -66,7 +66,7 @@ func RemoveFromAlbumCoverCache(uid string) { } if err := query.UpdateAlbumPreviews(); err != nil { - log.Errorf("failed updating album previews: %s", err) + log.Error(err) } } @@ -75,7 +75,7 @@ func FlushCoverCache() { service.CoverCache().Flush() if err := query.UpdatePreviews(); err != nil { - log.Errorf("failed updating preview images: %s", err) + log.Error(err) } log.Debugf("albums: flushed cover cache") diff --git a/internal/api/subject.go b/internal/api/subject.go index 3bdfb61c8..7e34e194f 100644 --- a/internal/api/subject.go +++ b/internal/api/subject.go @@ -99,7 +99,7 @@ func UpdateSubject(router *gin.RouterGroup) { return } - if txt.NameSlug(f.SubjName) == "" { + if txt.Slug(f.SubjName) == "" { // Return unchanged model data if (normalized) name is empty. c.JSON(http.StatusOK, m) return diff --git a/internal/classify/gen.go b/internal/classify/gen.go index 4b19cb9cd..962a73e7f 100644 --- a/internal/classify/gen.go +++ b/internal/classify/gen.go @@ -86,7 +86,7 @@ func main() { var packageTemplate = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT. package classify -var rules = LabelRules{ +var Rules = LabelRules{ {{- range $key, $value := .Rules }} {{ printf "%q" $key }}: { Label: {{ printf "%q" $value.Label }}, diff --git a/internal/classify/label.go b/internal/classify/label.go index 2dd27ca95..3affc4d4f 100644 --- a/internal/classify/label.go +++ b/internal/classify/label.go @@ -3,8 +3,6 @@ package classify import ( "strings" - "github.com/photoprism/photoprism/internal/face" - "github.com/photoprism/photoprism/pkg/txt" ) @@ -31,7 +29,7 @@ func LocationLabel(name string, uncertainty int) Label { var categories []string - if rule, ok := rules.Find(name); ok { + if rule, ok := Rules.Find(name); ok { priority = rule.Priority categories = rule.Categories } @@ -49,26 +47,3 @@ func LocationLabel(name string, uncertainty int) Label { func (l Label) Title() string { return txt.Title(txt.Clip(l.Name, txt.ClipDefault)) } - -// FaceLabels returns matching labels if there are people in the image. -func FaceLabels(faces face.Faces, src string) Labels { - var r LabelRule - - count := faces.Count() - - if count < 1 { - return Labels{} - } else if count == 1 { - r = rules["portrait"] - } else { - r = rules["people"] - } - - return Labels{Label{ - Name: r.Label, - Source: src, - Uncertainty: faces.Uncertainty(), - Priority: r.Priority, - Categories: r.Categories, - }} -} diff --git a/internal/classify/label_test.go b/internal/classify/label_test.go index e3061601a..397d03003 100644 --- a/internal/classify/label_test.go +++ b/internal/classify/label_test.go @@ -3,8 +3,6 @@ package classify import ( "testing" - "github.com/photoprism/photoprism/internal/face" - "github.com/stretchr/testify/assert" ) @@ -53,50 +51,3 @@ func TestLabel_Title(t *testing.T) { assert.Equal(t, "Berlin / Neukölln Hasenheide", LocLabel.Title()) }) } - -func TestFaceLabels(t *testing.T) { - Face1 := face.Face{ - Rows: 0, - Cols: 0, - Score: 0, - Area: face.Area{}, - Eyes: nil, - Landmarks: nil, - Embeddings: nil, - } - Face2 := face.Face{ - Rows: 0, - Cols: 0, - Score: 0, - Area: face.Area{}, - Eyes: nil, - Landmarks: nil, - Embeddings: nil, - } - t.Run("count < 1", func(t *testing.T) { - Faces := face.Faces{} - FaceLabels := FaceLabels(Faces, "") - t.Log(FaceLabels) - assert.Equal(t, 0, FaceLabels.Len()) - }) - t.Run("count > 1", func(t *testing.T) { - Faces := face.Faces{Face1, Face2} - FaceLabels := FaceLabels(Faces, "") - t.Log(FaceLabels) - assert.Equal(t, "people", FaceLabels[0].Name) - assert.Equal(t, "", FaceLabels[0].Source) - assert.Equal(t, 50, FaceLabels[0].Uncertainty) - assert.Equal(t, 0, FaceLabels[0].Priority) - //assert.Equal(t, "", FaceLabels[0].Categories) - }) - t.Run("count = 1", func(t *testing.T) { - Faces := face.Faces{Face1} - FaceLabels := FaceLabels(Faces, "test") - t.Log(FaceLabels) - assert.Equal(t, "portrait", FaceLabels[0].Name) - assert.Equal(t, "test", FaceLabels[0].Source) - assert.Equal(t, 50, FaceLabels[0].Uncertainty) - assert.Equal(t, 0, FaceLabels[0].Priority) - assert.Equal(t, "people", FaceLabels[0].Categories[0]) - }) -} diff --git a/internal/classify/rules.go b/internal/classify/rules.go index 0c4c94587..54ec3ae1a 100644 --- a/internal/classify/rules.go +++ b/internal/classify/rules.go @@ -1,7 +1,7 @@ // Code generated by go generate; DO NOT EDIT. package classify -var rules = LabelRules{ +var Rules = LabelRules{ "abacus": { Label: "", Threshold: 1.000000, diff --git a/internal/classify/rules_test.go b/internal/classify/rules_test.go index b90a652e6..f12a0669a 100644 --- a/internal/classify/rules_test.go +++ b/internal/classify/rules_test.go @@ -7,7 +7,7 @@ import ( ) func TestLabelRules_Find(t *testing.T) { - result, ok := rules.Find("cat") + result, ok := Rules.Find("cat") assert.True(t, ok) assert.Equal(t, "cat", result.Label) assert.Equal(t, "animal", result.Categories[0]) diff --git a/internal/classify/tensorflow.go b/internal/classify/tensorflow.go index 8d4a926a8..d815f7ea8 100644 --- a/internal/classify/tensorflow.go +++ b/internal/classify/tensorflow.go @@ -180,7 +180,7 @@ func (t *TensorFlow) bestLabels(probabilities []float32) Labels { labelText := strings.ToLower(t.labels[i]) - rule, _ := rules.Find(labelText) + rule, _ := Rules.Find(labelText) // discard labels that don't met the threshold if p < rule.Threshold { diff --git a/internal/entity/account.go b/internal/entity/account.go index 4f6e93f86..663f2d23d 100644 --- a/internal/entity/account.go +++ b/internal/entity/account.go @@ -9,6 +9,7 @@ import ( "github.com/photoprism/photoprism/internal/remote" "github.com/photoprism/photoprism/internal/remote/webdav" "github.com/photoprism/photoprism/pkg/fs" + "github.com/photoprism/photoprism/pkg/txt" "github.com/ulule/deepcopier" ) @@ -24,8 +25,8 @@ type Accounts []Account // Account represents a remote service account for uploading, downloading or syncing media files. type Account struct { ID uint `gorm:"primary_key"` - AccName string `gorm:"type:VARCHAR(255);"` - AccOwner string `gorm:"type:VARCHAR(255);"` + AccName string `gorm:"type:VARCHAR(160);"` + AccOwner string `gorm:"type:VARCHAR(160);"` AccURL string `gorm:"type:VARBINARY(512);"` AccType string `gorm:"type:VARBINARY(255);"` AccKey string `gorm:"type:VARBINARY(255);"` @@ -66,7 +67,7 @@ func CreateAccount(form form.Account) (model *Account, err error) { return model, err } -// Saves the entity using form data and stores it in the database. +// SaveForm saves the entity using form data and stores it in the database. func (m *Account) SaveForm(form form.Account) error { db := Db() @@ -94,6 +95,9 @@ func (m *Account) SaveForm(form form.Account) error { m.SyncStatus = AccountSyncStatusRefresh } + m.AccName = txt.Clip(m.AccName, txt.ClipName) + m.AccOwner = txt.Clip(m.AccOwner, txt.ClipName) + return db.Save(m).Error } @@ -119,7 +123,7 @@ func (m *Account) Updates(values interface{}) error { return UnscopedDb().Model(m).UpdateColumns(values).Error } -// Updates a column in the database. +// Update a column in the database. func (m *Account) Update(attr string, value interface{}) error { return UnscopedDb().Model(m).UpdateColumn(attr, value).Error } diff --git a/internal/entity/album.go b/internal/entity/album.go index 8f388cf1e..497b3cfc8 100644 --- a/internal/entity/album.go +++ b/internal/entity/album.go @@ -31,12 +31,12 @@ type Album struct { ID uint `gorm:"primary_key" json:"ID" yaml:"-"` AlbumUID string `gorm:"type:VARBINARY(42);unique_index;" json:"UID" yaml:"UID"` ParentUID string `gorm:"type:VARBINARY(42);default:'';" json:"ParentUID,omitempty" yaml:"ParentUID,omitempty"` - AlbumSlug string `gorm:"type:VARBINARY(255);index;" json:"Slug" yaml:"Slug"` + AlbumSlug string `gorm:"type:VARBINARY(160);index;" json:"Slug" yaml:"Slug"` AlbumPath string `gorm:"type:VARBINARY(500);index;" json:"Path,omitempty" yaml:"Path,omitempty"` AlbumType string `gorm:"type:VARBINARY(8);default:'album';" json:"Type" yaml:"Type,omitempty"` - AlbumTitle string `gorm:"type:VARCHAR(255);index;" json:"Title" yaml:"Title"` - AlbumLocation string `gorm:"type:VARCHAR(255);" json:"Location" yaml:"Location,omitempty"` - AlbumCategory string `gorm:"type:VARCHAR(255);index;" json:"Category" yaml:"Category,omitempty"` + AlbumTitle string `gorm:"type:VARCHAR(160);index;" json:"Title" yaml:"Title"` + AlbumLocation string `gorm:"type:VARCHAR(160);" json:"Location" yaml:"Location,omitempty"` + AlbumCategory string `gorm:"type:VARCHAR(100);index;" json:"Category" yaml:"Category,omitempty"` AlbumCaption string `gorm:"type:TEXT;" json:"Caption" yaml:"Caption,omitempty"` AlbumDescription string `gorm:"type:TEXT;" json:"Description" yaml:"Description,omitempty"` AlbumNotes string `gorm:"type:TEXT;" json:"Notes" yaml:"Notes,omitempty"` @@ -292,7 +292,7 @@ func (m *Album) String() string { return "[unknown album]" } -// Checks if the album is of type moment. +// IsMoment tests if the album is of type moment. func (m *Album) IsMoment() bool { return m.AlbumType == AlbumMoment } @@ -309,9 +309,9 @@ func (m *Album) SetTitle(title string) { if m.AlbumType == AlbumDefault { if len(m.AlbumTitle) < txt.ClipSlug { - m.AlbumSlug = slug.Make(m.AlbumTitle) + m.AlbumSlug = txt.Slug(m.AlbumTitle) } else { - m.AlbumSlug = slug.Make(txt.Clip(m.AlbumTitle, txt.ClipSlug)) + "-" + m.AlbumUID + m.AlbumSlug = txt.Slug(m.AlbumTitle) + "-" + m.AlbumUID } } @@ -327,7 +327,7 @@ func (m *Album) SaveForm(f form.Album) error { } if f.AlbumCategory != "" { - m.AlbumCategory = txt.Title(txt.Clip(f.AlbumCategory, txt.ClipKeyword)) + m.AlbumCategory = txt.Clip(txt.Title(f.AlbumCategory), txt.ClipCategory) } if f.AlbumTitle != "" { diff --git a/internal/entity/camera.go b/internal/entity/camera.go index 1a46047b8..9203f2ab0 100644 --- a/internal/entity/camera.go +++ b/internal/entity/camera.go @@ -5,7 +5,6 @@ import ( "sync" "time" - "github.com/gosimple/slug" "github.com/photoprism/photoprism/internal/event" "github.com/photoprism/photoprism/pkg/txt" ) @@ -18,11 +17,11 @@ type Cameras []Camera // Camera model and make (as extracted from UpdateExif metadata) type Camera struct { ID uint `gorm:"primary_key" json:"ID" yaml:"ID"` - CameraSlug string `gorm:"type:VARBINARY(255);unique_index;" json:"Slug" yaml:"-"` - CameraName string `gorm:"type:VARCHAR(255);" json:"Name" yaml:"Name"` - CameraMake string `gorm:"type:VARCHAR(255);" json:"Make" yaml:"Make,omitempty"` - CameraModel string `gorm:"type:VARCHAR(255);" json:"Model" yaml:"Model,omitempty"` - CameraType string `gorm:"type:VARCHAR(255);" json:"Type,omitempty" yaml:"Type,omitempty"` + CameraSlug string `gorm:"type:VARBINARY(160);unique_index;" json:"Slug" yaml:"-"` + CameraName string `gorm:"type:VARCHAR(160);" json:"Name" yaml:"Name"` + CameraMake string `gorm:"type:VARCHAR(160);" json:"Make" yaml:"Make,omitempty"` + CameraModel string `gorm:"type:VARCHAR(160);" json:"Model" yaml:"Model,omitempty"` + CameraType string `gorm:"type:VARCHAR(100);" json:"Type,omitempty" yaml:"Type,omitempty"` CameraDescription string `gorm:"type:TEXT;" json:"Description,omitempty" yaml:"Description,omitempty"` CameraNotes string `gorm:"type:TEXT;" json:"Notes,omitempty" yaml:"Notes,omitempty"` CreatedAt time.Time `json:"-" yaml:"-"` @@ -44,8 +43,8 @@ func CreateUnknownCamera() { // NewCamera creates a camera entity from a model name and a make name. func NewCamera(modelName string, makeName string) *Camera { - modelName = txt.Clip(modelName, txt.ClipDefault) - makeName = txt.Clip(makeName, txt.ClipDefault) + modelName = strings.TrimSpace(modelName) + makeName = strings.TrimSpace(makeName) if modelName == "" && makeName == "" { return &UnknownCamera @@ -72,13 +71,12 @@ func NewCamera(modelName string, makeName string) *Camera { } cameraName := strings.Join(name, " ") - cameraSlug := slug.Make(txt.Clip(cameraName, txt.ClipSlug)) result := &Camera{ - CameraSlug: cameraSlug, - CameraName: cameraName, - CameraMake: makeName, - CameraModel: modelName, + CameraSlug: txt.Slug(cameraName), + CameraName: txt.Clip(cameraName, txt.ClipName), + CameraMake: txt.Clip(makeName, txt.ClipName), + CameraModel: txt.Clip(modelName, txt.ClipName), } return result diff --git a/internal/entity/country.go b/internal/entity/country.go index a5309231f..82a819920 100644 --- a/internal/entity/country.go +++ b/internal/entity/country.go @@ -1,10 +1,10 @@ package entity import ( - "github.com/gosimple/slug" "github.com/jinzhu/gorm" "github.com/photoprism/photoprism/internal/event" "github.com/photoprism/photoprism/internal/maps" + "github.com/photoprism/photoprism/pkg/txt" ) // altCountryNames defines mapping between different names for the same country @@ -20,8 +20,8 @@ type Countries []Country // Country represents a country location, used for labeling photos. type Country struct { ID string `gorm:"type:VARBINARY(2);primary_key" json:"ID" yaml:"ID"` - CountrySlug string `gorm:"type:VARBINARY(255);unique_index;" json:"Slug" yaml:"-"` - CountryName string `json:"Name" yaml:"Name,omitempty"` + CountrySlug string `gorm:"type:VARBINARY(160);unique_index;" json:"Slug" yaml:"-"` + CountryName string `gorm:"type:VARCHAR(160);" json:"Name" yaml:"Name,omitempty"` CountryDescription string `gorm:"type:TEXT;" json:"Description,omitempty" yaml:"Description,omitempty"` CountryNotes string `gorm:"type:TEXT;" json:"Notes,omitempty" yaml:"Notes,omitempty"` CountryPhoto *Photo `json:"-" yaml:"-"` @@ -51,12 +51,10 @@ func NewCountry(countryCode string, countryName string) *Country { countryName = altName } - countrySlug := slug.MakeLang(countryName, "en") - result := &Country{ ID: countryCode, - CountryName: countryName, - CountrySlug: countrySlug, + CountryName: txt.Clip(countryName, txt.ClipName), + CountrySlug: txt.Slug(countryName), } return result diff --git a/internal/entity/details.go b/internal/entity/details.go index 923d6f413..3517de913 100644 --- a/internal/entity/details.go +++ b/internal/entity/details.go @@ -10,6 +10,9 @@ import ( var photoDetailsMutex = sync.Mutex{} +// ClipDetail is the size of a Details database column in runes. +const ClipDetail = 250 + // Details stores additional metadata fields for each photo to improve search performance. type Details struct { PhotoID uint `gorm:"primary_key;auto_increment:false" yaml:"-"` @@ -17,13 +20,13 @@ type Details struct { KeywordsSrc string `gorm:"type:VARBINARY(8);" json:"KeywordsSrc" yaml:"KeywordsSrc,omitempty"` Notes string `gorm:"type:TEXT;" json:"Notes" yaml:"Notes,omitempty"` NotesSrc string `gorm:"type:VARBINARY(8);" json:"NotesSrc" yaml:"NotesSrc,omitempty"` - Subject string `gorm:"type:VARCHAR(255);" json:"Subject" yaml:"Subject,omitempty"` + Subject string `gorm:"type:VARCHAR(250);" json:"Subject" yaml:"Subject,omitempty"` SubjectSrc string `gorm:"type:VARBINARY(8);" json:"SubjectSrc" yaml:"SubjectSrc,omitempty"` - Artist string `gorm:"type:VARCHAR(255);" json:"Artist" yaml:"Artist,omitempty"` + Artist string `gorm:"type:VARCHAR(250);" json:"Artist" yaml:"Artist,omitempty"` ArtistSrc string `gorm:"type:VARBINARY(8);" json:"ArtistSrc" yaml:"ArtistSrc,omitempty"` - Copyright string `gorm:"type:VARCHAR(255);" json:"Copyright" yaml:"Copyright,omitempty"` + Copyright string `gorm:"type:VARCHAR(250);" json:"Copyright" yaml:"Copyright,omitempty"` CopyrightSrc string `gorm:"type:VARBINARY(8);" json:"CopyrightSrc" yaml:"CopyrightSrc,omitempty"` - License string `gorm:"type:VARCHAR(255);" json:"License" yaml:"License,omitempty"` + License string `gorm:"type:VARCHAR(250);" json:"License" yaml:"License,omitempty"` LicenseSrc string `gorm:"type:VARBINARY(8);" json:"LicenseSrc" yaml:"LicenseSrc,omitempty"` CreatedAt time.Time `yaml:"-"` UpdatedAt time.Time `yaml:"-"` @@ -160,7 +163,7 @@ func (m *Details) SetKeywords(data, src string) { // SetSubject updates the photo details field. func (m *Details) SetSubject(data, src string) { - val := txt.Clip(data, txt.ClipVarchar) + val := txt.Clip(data, ClipDetail) if val == "" { return @@ -192,7 +195,7 @@ func (m *Details) SetNotes(data, src string) { // SetArtist updates the photo details field. func (m *Details) SetArtist(data, src string) { - val := txt.Clip(data, txt.ClipVarchar) + val := txt.Clip(data, ClipDetail) if val == "" { return @@ -208,7 +211,7 @@ func (m *Details) SetArtist(data, src string) { // SetCopyright updates the photo details field. func (m *Details) SetCopyright(data, src string) { - val := txt.Clip(data, txt.ClipVarchar) + val := txt.Clip(data, ClipDetail) if val == "" { return @@ -224,7 +227,7 @@ func (m *Details) SetCopyright(data, src string) { // SetLicense updates the photo details field. func (m *Details) SetLicense(data, src string) { - val := txt.Clip(data, txt.ClipVarchar) + val := txt.Clip(data, ClipDetail) if val == "" { return diff --git a/internal/entity/entity.go b/internal/entity/entity.go index 5f85d03d6..1ab90c30b 100644 --- a/internal/entity/entity.go +++ b/internal/entity/entity.go @@ -13,6 +13,8 @@ import ( "fmt" "time" + "github.com/photoprism/photoprism/pkg/txt" + "github.com/jinzhu/gorm" "github.com/photoprism/photoprism/internal/event" ) @@ -71,10 +73,10 @@ func (list Types) WaitForMigration() { for i := 0; i <= attempts; i++ { count := RowCount{} if err := Db().Raw(fmt.Sprintf("SELECT COUNT(*) AS count FROM %s", name)).Scan(&count).Error; err == nil { - // log.Debugf("entity: table %s migrated", name) + log.Tracef("entity: %s migrated", txt.Quote(name)) break } else { - log.Debugf("entity: wait for migration %s (%s)", err.Error(), name) + log.Debugf("entity: waiting for %s migration (%s)", txt.Quote(name), err.Error()) } if i == attempts { @@ -93,20 +95,21 @@ func (list Types) Truncate() { // log.Debugf("entity: removed all data from %s", name) break } else if err.Error() != "record not found" { - log.Debugf("entity: %s in %s", err, name) + log.Debugf("entity: %s in %s", err, txt.Quote(name)) } } } // Migrate migrates all database tables of registered entities. func (list Types) Migrate() { - for _, entity := range list { + for name, entity := range list { if err := UnscopedDb().AutoMigrate(entity).Error; err != nil { - log.Debugf("entity: migrate %s (waiting 1s)", err.Error()) + log.Debugf("entity: %s (waiting 1s)", err.Error()) time.Sleep(time.Second) if err := UnscopedDb().AutoMigrate(entity).Error; err != nil { + log.Errorf("entity: failed migrating %s", txt.Quote(name)) panic(err) } } diff --git a/internal/entity/face.go b/internal/entity/face.go index e6e0794cf..b8d82a4f4 100644 --- a/internal/entity/face.go +++ b/internal/entity/face.go @@ -351,8 +351,8 @@ func FindFace(id string) *Face { return &f } -// FaceCount counts the number of valid face markers for a file uid. -func FaceCount(fileUID string) (c int) { +// ValidFaceCount counts the number of valid face markers for a file uid. +func ValidFaceCount(fileUID string) (c int) { if !rnd.IsPPID(fileUID, 'f') { return } diff --git a/internal/entity/file.go b/internal/entity/file.go index 9adfa948a..4efcbcd52 100644 --- a/internal/entity/file.go +++ b/internal/entity/file.go @@ -445,9 +445,9 @@ func (m *File) AddFace(f face.Face, subjUID string) { } } -// FaceCount returns the current number of valid faces detected. -func (m *File) FaceCount() (c int) { - return FaceCount(m.FileUID) +// ValidFaceCount returns the number of valid face markers. +func (m *File) ValidFaceCount() (c int) { + return ValidFaceCount(m.FileUID) } // UpdatePhotoFaceCount updates the faces count in the index and returns it if the file is primary. @@ -457,7 +457,7 @@ func (m *File) UpdatePhotoFaceCount() (c int, err error) { return 0, nil } - c = m.FaceCount() + c = m.ValidFaceCount() err = UnscopedDb().Model(Photo{}). Where("id = ?", m.PhotoID). @@ -491,6 +491,15 @@ func (m *File) Markers() *Markers { return m.markers } +// UnsavedMarkers tests if any marker hasn't been saved yet. +func (m *File) UnsavedMarkers() bool { + if m.markers == nil { + return false + } + + return m.markers.Unsaved() +} + // SubjectNames returns all known subject names. func (m *File) SubjectNames() []string { return m.Markers().SubjectNames() diff --git a/internal/entity/file_test.go b/internal/entity/file_test.go index 88243a46f..323fd02da 100644 --- a/internal/entity/file_test.go +++ b/internal/entity/file_test.go @@ -506,11 +506,11 @@ func TestFile_AddFaces(t *testing.T) { }) } -func TestFile_FaceCount(t *testing.T) { +func TestFile_ValidFaceCount(t *testing.T) { t.Run("FileFixturesExampleBridge", func(t *testing.T) { file := FileFixturesExampleBridge - result := file.FaceCount() + result := file.ValidFaceCount() assert.GreaterOrEqual(t, result, 3) }) @@ -589,3 +589,29 @@ func TestFile_SubjectNames(t *testing.T) { } }) } + +func TestFile_UnsavedMarkers(t *testing.T) { + t.Run("bridge2.jpg", func(t *testing.T) { + m := FileFixtures.Get("bridge2.jpg") + assert.Equal(t, "ft2es49w15bnlqdw", m.FileUID) + assert.False(t, m.UnsavedMarkers()) + + markers := m.Markers() + + assert.Equal(t, 1, m.ValidFaceCount()) + assert.Equal(t, 1, markers.ValidFaceCount()) + assert.Equal(t, 1, markers.DetectedFaceCount()) + assert.False(t, m.UnsavedMarkers()) + assert.False(t, markers.Unsaved()) + + newMarker := *NewMarker(m, cropArea1, "lt9k3pw1wowuy1c1", SrcManual, MarkerFace, 100, 65) + + markers.Append(newMarker) + + assert.Equal(t, 1, m.ValidFaceCount()) + assert.Equal(t, 2, markers.ValidFaceCount()) + assert.Equal(t, 1, markers.DetectedFaceCount()) + assert.True(t, m.UnsavedMarkers()) + assert.True(t, markers.Unsaved()) + }) +} diff --git a/internal/entity/folder.go b/internal/entity/folder.go index 2f5c5e9c5..dca87ffe5 100644 --- a/internal/entity/folder.go +++ b/internal/entity/folder.go @@ -7,7 +7,6 @@ import ( "sync" "time" - "github.com/gosimple/slug" "github.com/jinzhu/gorm" "github.com/photoprism/photoprism/internal/form" "github.com/photoprism/photoprism/pkg/rnd" @@ -25,8 +24,8 @@ type Folder struct { Root string `gorm:"type:VARBINARY(16);default:'';unique_index:idx_folders_path_root;" json:"Root" yaml:"Root,omitempty"` FolderUID string `gorm:"type:VARBINARY(42);primary_key;" json:"UID,omitempty" yaml:"UID,omitempty"` FolderType string `gorm:"type:VARBINARY(16);" json:"Type" yaml:"Type,omitempty"` - FolderTitle string `gorm:"type:VARCHAR(255);" json:"Title" yaml:"Title,omitempty"` - FolderCategory string `gorm:"type:VARCHAR(255);index;" json:"Category" yaml:"Category,omitempty"` + FolderTitle string `gorm:"type:VARCHAR(200);" json:"Title" yaml:"Title,omitempty"` + FolderCategory string `gorm:"type:VARCHAR(100);index;" json:"Category" yaml:"Category,omitempty"` FolderDescription string `gorm:"type:TEXT;" json:"Description,omitempty" yaml:"Description,omitempty"` FolderOrder string `gorm:"type:VARBINARY(32);" json:"Order" yaml:"Order,omitempty"` FolderCountry string `gorm:"type:VARBINARY(2);index:idx_folders_country_year_month;default:'zz'" json:"Country" yaml:"Country,omitempty"` @@ -130,13 +129,13 @@ func (m *Folder) SetValuesFromPath() { } if m.FolderTitle == "" { - m.FolderTitle = txt.Title(s) + m.FolderTitle = txt.Clip(txt.Title(s), txt.ClipTitle) } } // Slug returns a slug based on the folder title. func (m *Folder) Slug() string { - return slug.Make(m.Path) + return txt.Slug(m.Path) } // RootPath returns the full folder path including root. @@ -144,12 +143,12 @@ func (m *Folder) RootPath() string { return path.Join(m.Root, m.Path) } -// Title returns a human readable folder title. +// Title returns the human-readable folder title. func (m *Folder) Title() string { return m.FolderTitle } -// Saves the complete entity in the database. +// Create inserts the entity to the index. func (m *Folder) Create() error { folderMutex.Lock() defer folderMutex.Unlock() @@ -232,5 +231,8 @@ func (m *Folder) SetForm(f form.Folder) error { return err } + m.FolderTitle = txt.Clip(m.FolderTitle, txt.ClipTitle) + m.FolderCategory = txt.Clip(m.FolderCategory, txt.ClipCategory) + return nil } diff --git a/internal/entity/label.go b/internal/entity/label.go index a1f08dbb6..e558845ec 100644 --- a/internal/entity/label.go +++ b/internal/entity/label.go @@ -4,7 +4,6 @@ import ( "sync" "time" - "github.com/gosimple/slug" "github.com/jinzhu/gorm" "github.com/photoprism/photoprism/internal/classify" "github.com/photoprism/photoprism/internal/event" @@ -20,9 +19,9 @@ type Labels []Label type Label struct { ID uint `gorm:"primary_key" json:"ID" yaml:"-"` LabelUID string `gorm:"type:VARBINARY(42);unique_index;" json:"UID" yaml:"UID"` - LabelSlug string `gorm:"type:VARBINARY(255);unique_index;" json:"Slug" yaml:"-"` - CustomSlug string `gorm:"type:VARBINARY(255);index;" json:"CustomSlug" yaml:"-"` - LabelName string `gorm:"type:VARCHAR(255);" json:"Name" yaml:"Name"` + LabelSlug string `gorm:"type:VARBINARY(160);unique_index;" json:"Slug" yaml:"-"` + CustomSlug string `gorm:"type:VARBINARY(160);index;" json:"CustomSlug" yaml:"-"` + LabelName string `gorm:"type:VARCHAR(160);" json:"Name" yaml:"Name"` LabelPriority int `json:"Priority" yaml:"Priority,omitempty"` LabelFavorite bool `json:"Favorite" yaml:"Favorite,omitempty"` LabelDescription string `gorm:"type:TEXT;" json:"Description" yaml:"Description,omitempty"` @@ -60,12 +59,12 @@ func NewLabel(name string, priority int) *Label { } labelName = txt.Title(labelName) - labelSlug := slug.Make(txt.Clip(labelName, txt.ClipSlug)) + labelSlug := txt.Slug(labelName) result := &Label{ LabelSlug: labelSlug, CustomSlug: labelSlug, - LabelName: labelName, + LabelName: txt.Clip(labelName, txt.ClipName), LabelPriority: priority, PhotoCount: 1, } @@ -142,7 +141,7 @@ func FirstOrCreateLabel(m *Label) *Label { // FindLabel returns an existing row if exists. func FindLabel(s string) *Label { - labelSlug := slug.Make(txt.Clip(s, txt.ClipSlug)) + labelSlug := txt.Slug(s) result := Label{} @@ -167,8 +166,8 @@ func (m *Label) SetName(name string) { return } - m.LabelName = name - m.CustomSlug = txt.NameSlug(name) + m.LabelName = txt.Clip(name, txt.ClipName) + m.CustomSlug = txt.Slug(name) } // UpdateClassify updates a label if necessary diff --git a/internal/entity/lens.go b/internal/entity/lens.go index 35f246196..204393b73 100644 --- a/internal/entity/lens.go +++ b/internal/entity/lens.go @@ -5,7 +5,6 @@ import ( "sync" "time" - "github.com/gosimple/slug" "github.com/photoprism/photoprism/internal/event" "github.com/photoprism/photoprism/pkg/txt" ) @@ -18,11 +17,11 @@ type Lenses []Lens // Lens represents camera lens (as extracted from UpdateExif metadata) type Lens struct { ID uint `gorm:"primary_key" json:"ID" yaml:"ID"` - LensSlug string `gorm:"type:VARBINARY(255);unique_index;" json:"Slug" yaml:"Slug,omitempty"` - LensName string `gorm:"type:VARCHAR(255);" json:"Name" yaml:"Name"` - LensMake string `gorm:"type:VARCHAR(255);" json:"Make" yaml:"Make,omitempty"` - LensModel string `gorm:"type:VARCHAR(255);" json:"Model" yaml:"Model,omitempty"` - LensType string `gorm:"type:VARCHAR(255);" json:"Type" yaml:"Type,omitempty"` + LensSlug string `gorm:"type:VARBINARY(160);unique_index;" json:"Slug" yaml:"Slug,omitempty"` + LensName string `gorm:"type:VARCHAR(160);" json:"Name" yaml:"Name"` + LensMake string `gorm:"type:VARCHAR(160);" json:"Make" yaml:"Make,omitempty"` + LensModel string `gorm:"type:VARCHAR(160);" json:"Model" yaml:"Model,omitempty"` + LensType string `gorm:"type:VARCHAR(100);" json:"Type" yaml:"Type,omitempty"` LensDescription string `gorm:"type:TEXT;" json:"Description,omitempty" yaml:"Description,omitempty"` LensNotes string `gorm:"type:TEXT;" json:"Notes,omitempty" yaml:"Notes,omitempty"` CreatedAt time.Time `json:"-" yaml:"-"` @@ -49,8 +48,8 @@ func (Lens) TableName() string { // NewLens creates a new lens in database func NewLens(modelName string, makeName string) *Lens { - modelName = txt.Clip(modelName, txt.ClipDefault) - makeName = txt.Clip(makeName, txt.ClipDefault) + modelName = strings.TrimSpace(modelName) + makeName = strings.TrimSpace(makeName) if modelName == "" && makeName == "" { return &UnknownLens @@ -73,13 +72,12 @@ func NewLens(modelName string, makeName string) *Lens { } lensName := strings.Join(name, " ") - lensSlug := slug.Make(txt.Clip(lensName, txt.ClipSlug)) result := &Lens{ - LensSlug: lensSlug, - LensName: lensName, - LensMake: makeName, - LensModel: modelName, + LensSlug: txt.Slug(lensName), + LensName: txt.Clip(lensName, txt.ClipName), + LensMake: txt.Clip(makeName, txt.ClipName), + LensModel: txt.Clip(modelName, txt.ClipName), } return result diff --git a/internal/entity/link.go b/internal/entity/link.go index 17eae9c6b..73fc9d586 100644 --- a/internal/entity/link.go +++ b/internal/entity/link.go @@ -4,7 +4,6 @@ import ( "fmt" "time" - "github.com/gosimple/slug" "github.com/jinzhu/gorm" "github.com/photoprism/photoprism/pkg/rnd" @@ -17,8 +16,8 @@ type Links []Link type Link struct { LinkUID string `gorm:"type:VARBINARY(42);primary_key;" json:"UID,omitempty" yaml:"UID,omitempty"` ShareUID string `gorm:"type:VARBINARY(42);unique_index:idx_links_uid_token;" json:"Share" yaml:"Share"` - ShareSlug string `gorm:"type:VARBINARY(255);index;" json:"Slug" yaml:"Slug,omitempty"` - LinkToken string `gorm:"type:VARBINARY(255);unique_index:idx_links_uid_token;" json:"Token" yaml:"Token,omitempty"` + ShareSlug string `gorm:"type:VARBINARY(160);index;" json:"Slug" yaml:"Slug,omitempty"` + LinkToken string `gorm:"type:VARBINARY(160);unique_index:idx_links_uid_token;" json:"Token" yaml:"Token,omitempty"` LinkExpires int `json:"Expires" yaml:"Expires,omitempty"` LinkViews uint `json:"Views" yaml:"-"` MaxViews uint `json:"MaxViews" yaml:"-"` @@ -81,7 +80,7 @@ func (m *Link) Expired() bool { } func (m *Link) SetSlug(s string) { - m.ShareSlug = slug.Make(txt.Clip(s, txt.ClipSlug)) + m.ShareSlug = txt.Slug(s) } func (m *Link) SetPassword(password string) error { diff --git a/internal/entity/marker.go b/internal/entity/marker.go index 71579e386..3e0a6ff72 100644 --- a/internal/entity/marker.go +++ b/internal/entity/marker.go @@ -29,7 +29,7 @@ type Marker struct { FileUID string `gorm:"type:VARBINARY(42);index;default:'';" json:"FileUID" yaml:"FileUID"` MarkerType string `gorm:"type:VARBINARY(8);default:'';" json:"Type" yaml:"Type"` MarkerSrc string `gorm:"type:VARBINARY(8);default:'';" json:"Src" yaml:"Src,omitempty"` - MarkerName string `gorm:"type:VARCHAR(255);" json:"Name" yaml:"Name,omitempty"` + MarkerName string `gorm:"type:VARCHAR(160);" json:"Name" yaml:"Name,omitempty"` MarkerReview bool `json:"Review" yaml:"Review,omitempty"` MarkerInvalid bool `json:"Invalid" yaml:"Invalid,omitempty"` SubjUID string `gorm:"type:VARBINARY(42);index:idx_markers_subj_uid_src;" json:"SubjUID" yaml:"SubjUID,omitempty"` @@ -552,6 +552,49 @@ func (m *Marker) OverlapPercent(marker Marker) int { return int(math.Round(marker.SurfaceRatio(m.OverlapArea(marker)) * 100)) } +// Unsaved tests if the marker hasn't been saved yet. +func (m *Marker) Unsaved() bool { + return m.MarkerUID == "" || m.CreatedAt.IsZero() +} + +// ValidFace tests if the marker is a valid face. +func (m *Marker) ValidFace() bool { + return m.MarkerType == MarkerFace && !m.MarkerInvalid +} + +// DetectedFace tests if the marker is an automatically detected face. +func (m *Marker) DetectedFace() bool { + return m.MarkerType == MarkerFace && m.MarkerSrc == SrcImage +} + +// Uncertainty returns the detection uncertainty based on the score in percent. +func (m *Marker) Uncertainty() int { + switch { + case m.Score > 300: + return 1 + case m.Score > 200: + return 5 + case m.Score > 100: + return 10 + case m.Score > 80: + return 15 + case m.Score > 65: + return 20 + case m.Score > 50: + return 25 + case m.Score > 40: + return 30 + case m.Score > 30: + return 35 + case m.Score > 20: + return 40 + case m.Score > 10: + return 45 + } + + return 50 +} + // FindMarker returns an existing row if exists. func FindMarker(markerUid string) *Marker { if markerUid == "" { diff --git a/internal/entity/markers.go b/internal/entity/markers.go index b5c460915..ace59b348 100644 --- a/internal/entity/markers.go +++ b/internal/entity/markers.go @@ -1,6 +1,7 @@ package entity import ( + "github.com/photoprism/photoprism/internal/classify" "github.com/photoprism/photoprism/internal/face" "github.com/photoprism/photoprism/pkg/txt" ) @@ -26,6 +27,17 @@ func (m Markers) Save(file *File) (count int, err error) { return file.UpdatePhotoFaceCount() } +// Unsaved tests if any marker hasn't been saved yet. +func (m Markers) Unsaved() bool { + for _, marker := range m { + if marker.Unsaved() { + return true + } + } + + return false +} + // Contains returns true if a marker at the same position already exists. func (m Markers) Contains(other Marker) bool { for _, marker := range m { @@ -37,15 +49,26 @@ func (m Markers) Contains(other Marker) bool { return false } -// FaceCount returns the number of valid face markers. -func (m Markers) FaceCount() (faces int) { +// DetectedFaceCount returns the number of automatically detected face markers. +func (m Markers) DetectedFaceCount() (count int) { for _, marker := range m { - if !marker.MarkerInvalid && marker.MarkerType == MarkerFace { - faces++ + if marker.DetectedFace() { + count++ } } - return faces + return count +} + +// ValidFaceCount returns the number of valid face markers. +func (m Markers) ValidFaceCount() (count int) { + for _, marker := range m { + if marker.ValidFace() { + count++ + } + } + + return count } // SubjectNames returns known subject names. @@ -61,6 +84,48 @@ func (m Markers) SubjectNames() (names []string) { return txt.UniqueNames(names) } +// Labels returns matching labels. +func (m Markers) Labels() (result classify.Labels) { + faceCount := 0 + + labelSrc := SrcImage + labelUncertainty := 100 + + for _, marker := range m { + if marker.ValidFace() { + faceCount++ + + if u := marker.Uncertainty(); u < labelUncertainty { + labelUncertainty = u + } + + if marker.MarkerSrc != "" { + labelSrc = marker.MarkerSrc + } + } + } + + if faceCount < 1 { + return classify.Labels{} + } + + var rule classify.LabelRule + + if faceCount == 1 { + rule = classify.Rules["portrait"] + } else { + rule = classify.Rules["people"] + } + + return classify.Labels{classify.Label{ + Name: rule.Label, + Source: labelSrc, + Uncertainty: labelUncertainty, + Priority: rule.Priority, + Categories: rule.Categories, + }} +} + // Append adds a marker. func (m *Markers) Append(marker Marker) { *m = append(*m, marker) diff --git a/internal/entity/markers_test.go b/internal/entity/markers_test.go index efc40108c..d4480ec33 100644 --- a/internal/entity/markers_test.go +++ b/internal/entity/markers_test.go @@ -63,15 +63,26 @@ func TestMarkers_Contains(t *testing.T) { }) } -func TestMarkers_FaceCount(t *testing.T) { +func TestMarkers_DetectedFaceCount(t *testing.T) { m1 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea1, "lt9k3pw1wowuy1c1", SrcImage, MarkerFace, 100, 65) - m2 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea4, "lt9k3pw1wowuy1c2", SrcImage, MarkerFace, 100, 65) - m3 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea3, "lt9k3pw1wowuy1c3", SrcImage, MarkerFace, 100, 65) + m2 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea4, "lt9k3pw1wowuy1c2", SrcManual, MarkerFace, 100, 65) + m3 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea3, "lt9k3pw1wowuy1c3", SrcManual, MarkerFace, 100, 65) m3.MarkerInvalid = true m := Markers{m1, m2, m3} - assert.Equal(t, 2, m.FaceCount()) + assert.Equal(t, 1, m.DetectedFaceCount()) +} + +func TestMarkers_ValidFaceCount(t *testing.T) { + m1 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea1, "lt9k3pw1wowuy1c1", SrcImage, MarkerFace, 100, 65) + m2 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea4, "lt9k3pw1wowuy1c2", SrcManual, MarkerFace, 100, 65) + m3 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea3, "lt9k3pw1wowuy1c3", SrcManual, MarkerFace, 100, 65) + m3.MarkerInvalid = true + + m := Markers{m1, m2, m3} + + assert.Equal(t, 2, m.ValidFaceCount()) } func TestMarkers_SubjectNames(t *testing.T) { @@ -85,3 +96,63 @@ func TestMarkers_SubjectNames(t *testing.T) { assert.Equal(t, []string{"Jens Mander", "Corn McCornface"}, m.SubjectNames()) } + +func TestMarkers_Labels(t *testing.T) { + t.Run("None", func(t *testing.T) { + m := Markers{} + + result := m.Labels() + + if len(result) > 0 { + t.Fatalf("unexpected result: %#v", result) + } + }) + t.Run("One", func(t *testing.T) { + m1 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea1, "lt9k3pw1wowuy1c1", SrcImage, MarkerFace, 100, 12) + m2 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea4, "lt9k3pw1wowuy1c2", SrcImage, MarkerFace, 100, 300) + + m2.MarkerInvalid = true + + m := Markers{m1, m2} + + result := m.Labels() + + if len(result) == 1 { + t.Logf("labels: %#v", result) + + assert.Equal(t, "portrait", result[0].Name) + assert.Equal(t, SrcImage, result[0].Source) + assert.Equal(t, 45, result[0].Uncertainty) + assert.Equal(t, 0, result[0].Priority) + assert.Len(t, result[0].Categories, 1) + + if len(result[0].Categories) == 1 { + assert.Equal(t, "people", result[0].Categories[0]) + } + } else { + t.Fatalf("unexpected result: %#v", result) + } + }) + t.Run("Many", func(t *testing.T) { + m1 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea1, "lt9k3pw1wowuy1c1", SrcImage, MarkerFace, 100, 65) + m2 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea4, "lt9k3pw1wowuy1c2", SrcImage, MarkerFace, 100, 65) + m3 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea3, "lt9k3pw1wowuy1c3", SrcImage, MarkerFace, 100, 65) + m3.MarkerInvalid = true + + m := Markers{m1, m2, m3} + + result := m.Labels() + + if len(result) == 1 { + t.Logf("labels: %#v", result) + + assert.Equal(t, "people", result[0].Name) + assert.Equal(t, SrcImage, result[0].Source) + assert.Equal(t, 25, result[0].Uncertainty) + assert.Equal(t, 0, result[0].Priority) + assert.Len(t, result[0].Categories, 0) + } else { + t.Fatalf("unexpected result: %#v", result) + } + }) +} diff --git a/internal/entity/photo.go b/internal/entity/photo.go index d42badda5..672c355cb 100644 --- a/internal/entity/photo.go +++ b/internal/entity/photo.go @@ -49,7 +49,7 @@ type Photo struct { PhotoUID string `gorm:"type:VARBINARY(42);unique_index;index:idx_photos_taken_uid;" json:"UID" yaml:"UID"` PhotoType string `gorm:"type:VARBINARY(8);default:'image';" json:"Type" yaml:"Type"` TypeSrc string `gorm:"type:VARBINARY(8);" json:"TypeSrc" yaml:"TypeSrc,omitempty"` - PhotoTitle string `gorm:"type:VARCHAR(255);" json:"Title" yaml:"Title"` + PhotoTitle string `gorm:"type:VARCHAR(200);" json:"Title" yaml:"Title"` TitleSrc string `gorm:"type:VARBINARY(8);" json:"TitleSrc" yaml:"TitleSrc,omitempty"` PhotoDescription string `gorm:"type:TEXT;" json:"Description" yaml:"Description,omitempty"` DescriptionSrc string `gorm:"type:VARBINARY(8);" json:"DescriptionSrc" yaml:"DescriptionSrc,omitempty"` @@ -82,7 +82,7 @@ type Photo struct { PhotoResolution int `gorm:"type:SMALLINT" json:"Resolution" yaml:"-"` PhotoColor uint8 `json:"Color" yaml:"-"` CameraID uint `gorm:"index:idx_photos_camera_lens;default:1" json:"CameraID" yaml:"-"` - CameraSerial string `gorm:"type:VARBINARY(255);" json:"CameraSerial" yaml:"CameraSerial,omitempty"` + CameraSerial string `gorm:"type:VARBINARY(160);" json:"CameraSerial" yaml:"CameraSerial,omitempty"` CameraSrc string `gorm:"type:VARBINARY(8);" json:"CameraSrc" yaml:"-"` LensID uint `gorm:"index:idx_photos_camera_lens;default:1" json:"LensID" yaml:"-"` Details *Details `gorm:"association_autoupdate:false;association_autocreate:false;association_save_reference:false" json:"Details" yaml:"Details"` @@ -1073,8 +1073,8 @@ func (m *Photo) MapKey() string { // SetCameraSerial updates the camera serial number. func (m *Photo) SetCameraSerial(s string) { - if val := txt.Clip(s, txt.ClipVarchar); m.NoCameraSerial() && val != "" { - m.CameraSerial = val + if s = txt.Clip(s, txt.ClipDefault); m.NoCameraSerial() && s != "" { + m.CameraSerial = s } } @@ -1083,6 +1083,6 @@ func (m *Photo) FaceCount() int { if f, err := m.PrimaryFile(); err != nil { return 0 } else { - return f.FaceCount() + return f.ValidFaceCount() } } diff --git a/internal/entity/photo_counts.go b/internal/entity/photo_counts.go index 4334e3aaf..ef79c5fdb 100644 --- a/internal/entity/photo_counts.go +++ b/internal/entity/photo_counts.go @@ -2,6 +2,7 @@ package entity import ( "fmt" + "strings" "time" "github.com/jinzhu/gorm" @@ -146,10 +147,22 @@ func UpdateLabelPhotoCounts() (err error) { // UpdatePhotoCounts updates precalculated photo and file counts. func UpdatePhotoCounts() (err error) { if err = UpdatePlacesPhotoCounts(); err != nil { + if strings.Contains(err.Error(), "Error 1054") { + log.Errorf("counts: failed updating places, deprecated or unsupported database") + log.Tracef("counts: %s", err) + return nil + } + return err } if err = UpdateSubjectFileCounts(); err != nil { + if strings.Contains(err.Error(), "Error 1054") { + log.Errorf("counts: failed updating subjects, deprecated or unsupported database") + log.Tracef("counts: %s", err) + return nil + } + return err } diff --git a/internal/entity/photo_title.go b/internal/entity/photo_title.go index 717f33df2..81f105417 100644 --- a/internal/entity/photo_title.go +++ b/internal/entity/photo_title.go @@ -22,9 +22,9 @@ func (m *Photo) NoTitle() bool { // SetTitle changes the photo title and clips it to 300 characters. func (m *Photo) SetTitle(title, source string) { - newTitle := txt.Clip(title, txt.ClipDefault) + title = txt.Shorten(title, txt.ClipTitle, txt.Ellipsis) - if newTitle == "" { + if title == "" { return } @@ -32,7 +32,7 @@ func (m *Photo) SetTitle(title, source string) { return } - m.PhotoTitle = newTitle + m.PhotoTitle = title m.TitleSrc = source } diff --git a/internal/entity/subject.go b/internal/entity/subject.go index 8fa06a327..c0aff6545 100644 --- a/internal/entity/subject.go +++ b/internal/entity/subject.go @@ -3,10 +3,10 @@ package entity import ( "encoding/json" "fmt" + "strings" "sync" "time" - "github.com/gosimple/slug" "github.com/jinzhu/gorm" "github.com/photoprism/photoprism/internal/event" "github.com/photoprism/photoprism/pkg/rnd" @@ -23,17 +23,17 @@ type Subject struct { SubjUID string `gorm:"type:VARBINARY(42);primary_key;auto_increment:false;" json:"UID" yaml:"UID"` SubjType string `gorm:"type:VARBINARY(8);default:'';" json:"Type,omitempty" yaml:"Type,omitempty"` SubjSrc string `gorm:"type:VARBINARY(8);default:'';" json:"Src,omitempty" yaml:"Src,omitempty"` - SubjSlug string `gorm:"type:VARBINARY(255);index;default:'';" json:"Slug" yaml:"-"` - SubjName string `gorm:"type:VARCHAR(255);unique_index;default:'';" json:"Name" yaml:"Name"` - SubjAlias string `gorm:"type:VARCHAR(255);default:'';" json:"Alias" yaml:"Alias"` + SubjSlug string `gorm:"type:VARBINARY(160);index;default:'';" json:"Slug" yaml:"-"` + SubjName string `gorm:"type:VARCHAR(160);unique_index;default:'';" json:"Name" yaml:"Name"` + SubjAlias string `gorm:"type:VARCHAR(160);default:'';" json:"Alias" yaml:"Alias"` SubjBio string `gorm:"type:TEXT;" json:"Bio" yaml:"Bio,omitempty"` SubjNotes string `gorm:"type:TEXT;" json:"Notes,omitempty" yaml:"Notes,omitempty"` - SubjFavorite bool `gorm:"default:false" json:"Favorite" yaml:"Favorite,omitempty"` - SubjPrivate bool `gorm:"default:false" json:"Private" yaml:"Private,omitempty"` - SubjExcluded bool `gorm:"default:false" json:"Excluded" yaml:"Excluded,omitempty"` - FileCount int `gorm:"default:0" json:"FileCount" yaml:"-"` - Thumb string `gorm:"type:VARBINARY(128);index;default:''" json:"Thumb" yaml:"Thumb,omitempty"` - ThumbSrc string `gorm:"type:VARBINARY(8);default:''" json:"ThumbSrc,omitempty" yaml:"ThumbSrc,omitempty"` + SubjFavorite bool `gorm:"default:false;" json:"Favorite" yaml:"Favorite,omitempty"` + SubjPrivate bool `gorm:"default:false;" json:"Private" yaml:"Private,omitempty"` + SubjExcluded bool `gorm:"default:false;" json:"Excluded" yaml:"Excluded,omitempty"` + FileCount int `gorm:"default:0;" json:"FileCount" yaml:"-"` + Thumb string `gorm:"type:VARBINARY(128);index;default:'';" json:"Thumb" yaml:"Thumb,omitempty"` + ThumbSrc string `gorm:"type:VARBINARY(8);default:'';" json:"ThumbSrc,omitempty" yaml:"ThumbSrc,omitempty"` MetadataJSON json.RawMessage `gorm:"type:MEDIUMBLOB;" json:"Metadata,omitempty" yaml:"Metadata,omitempty"` CreatedAt time.Time `json:"CreatedAt" yaml:"-"` UpdatedAt time.Time `json:"UpdatedAt" yaml:"-"` @@ -56,26 +56,25 @@ func (m *Subject) BeforeCreate(scope *gorm.Scope) error { // NewSubject returns a new entity. func NewSubject(name, subjType, subjSrc string) *Subject { + // Name is required. + if strings.TrimSpace(name) == "" { + return nil + } + if subjType == "" { subjType = SubjPerson } - subjName := txt.Title(txt.Clip(name, txt.ClipDefault)) - subjSlug := slug.Make(txt.Clip(name, txt.ClipSlug)) - - // Name is required. - if subjName == "" || subjSlug == "" { - return nil - } - result := &Subject{ - SubjSlug: subjSlug, - SubjName: subjName, SubjType: subjType, SubjSrc: subjSrc, FileCount: 1, } + if err := result.SetName(name); err != nil { + log.Errorf("subject: %s", err) + } + return result } @@ -243,11 +242,11 @@ func (m *Subject) SetName(name string) error { name = txt.NormalizeName(name) if name == "" { - return fmt.Errorf("subject: name must not be empty") + return fmt.Errorf("name must not be empty") } m.SubjName = name - m.SubjSlug = txt.NameSlug(name) + m.SubjSlug = txt.Slug(name) return nil } diff --git a/internal/entity/subject_test.go b/internal/entity/subject_test.go index 5de4d3654..3a6162855 100644 --- a/internal/entity/subject_test.go +++ b/internal/entity/subject_test.go @@ -52,10 +52,12 @@ func TestSubject_SetName(t *testing.T) { assert.Equal(t, "jens-mander", m.SubjSlug) err := m.SetName("") + if err == nil { t.Fatal(err) } - assert.Equal(t, "subject: name must not be empty", err.Error()) + + assert.Equal(t, "name must not be empty", err.Error()) assert.Equal(t, "Jens Mander", m.SubjName) }) } diff --git a/internal/hub/feedback.go b/internal/hub/feedback.go index 8db0787de..4a3954f43 100644 --- a/internal/hub/feedback.go +++ b/internal/hub/feedback.go @@ -43,7 +43,7 @@ func NewFeedback(version, serial string) *Feedback { func (c *Config) SendFeedback(f form.Feedback) (err error) { feedback := NewFeedback(c.Version, c.Serial) feedback.Category = f.Category - feedback.Subject = txt.TrimLen(f.Message, 50) + feedback.Subject = txt.Shorten(f.Message, 50, "...") feedback.Message = f.Message feedback.UserName = f.UserName feedback.UserEmail = f.UserEmail diff --git a/internal/photoprism/index_mediafile.go b/internal/photoprism/index_mediafile.go index dcc014deb..dcbcfeb41 100644 --- a/internal/photoprism/index_mediafile.go +++ b/internal/photoprism/index_mediafile.go @@ -260,21 +260,29 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) ( result.Status = IndexSkipped return result } else if ind.findFaces && file.FilePrimary { - faces := ind.Faces(m, photo.PhotoFaces) + if markers := file.Markers(); markers != nil { + // Detect faces. + faces := ind.Faces(m, markers.DetectedFaceCount()) - if len(faces) > 0 { - file.AddFaces(faces) - } - - if c := file.Markers().FaceCount(); photo.PhotoFaces != c { - if c > photo.PhotoFaces { - extraLabels = append(extraLabels, classify.FaceLabels(faces, entity.SrcImage)...) + // Create markers from faces and add them. + if len(faces) > 0 { + file.AddFaces(faces) } - photo.PhotoFaces = c - } else if o.FacesOnly { - result.Status = IndexSkipped - return result + // Any new markers? + if file.UnsavedMarkers() { + // Add matching labels. + extraLabels = append(extraLabels, file.Markers().Labels()...) + } else if o.FacesOnly { + // Skip when indexing faces only. + result.Status = IndexSkipped + return result + } + + // Update photo face count. + photo.PhotoFaces = markers.ValidFaceCount() + } else { + log.Errorf("index: failed loading markers for %s", logName) } } diff --git a/internal/query/previews.go b/internal/query/previews.go index bba5ad557..f0cbf40c2 100644 --- a/internal/query/previews.go +++ b/internal/query/previews.go @@ -2,6 +2,7 @@ package query import ( "fmt" + "strings" "time" "github.com/jinzhu/gorm" @@ -21,7 +22,13 @@ func UpdateAlbumDefaultPreviews() (err error) { ORDER BY p.taken_at DESC LIMIT 1 ) WHERE thumb_src='' AND album_type = 'album' AND deleted_at IS NULL`)).Error - log.Debugf("previews: updated albums [%s]", time.Since(start)) + if err == nil { + log.Debugf("previews: updated albums [%s]", time.Since(start)) + } else if strings.Contains(err.Error(), "Error 1054") { + log.Errorf("previews: failed updating albums, deprecated or unsupported database") + log.Tracef("previews: %s", err) + return nil + } return err } @@ -39,7 +46,13 @@ func UpdateAlbumFolderPreviews() (err error) { ) WHERE thumb_src = '' AND album_type = 'folder' AND deleted_at IS NULL`)). Error - log.Debugf("previews: updated folders [%s]", time.Since(start)) + if err == nil { + log.Debugf("previews: updated folders [%s]", time.Since(start)) + } else if strings.Contains(err.Error(), "Error 1054") { + log.Errorf("previews: failed updating folders, deprecated or unsupported database") + log.Tracef("previews: %s", err) + return nil + } return err } @@ -80,7 +93,14 @@ func UpdateAlbumMonthPreviews() (err error) { return nil } */ - log.Debugf("previews: updated calendar [%s]", time.Since(start)) + + if err == nil { + log.Debugf("previews: updated calendar [%s]", time.Since(start)) + } else if strings.Contains(err.Error(), "Error 1054") { + log.Errorf("previews: failed updating calendar, deprecated or unsupported database") + log.Tracef("previews: %s", err) + return nil + } return err } @@ -110,7 +130,7 @@ func UpdateLabelPreviews() (err error) { start := time.Now() // Labels. - if err = Db().Table(entity.Label{}.TableName()). + err = Db().Table(entity.Label{}.TableName()). UpdateColumn("thumb", gorm.Expr(`( SELECT f.file_hash FROM files f JOIN photos_labels pl ON pl.label_id = labels.id AND pl.photo_id = f.photo_id AND pl.uncertainty < 100 @@ -118,13 +138,17 @@ func UpdateLabelPreviews() (err error) { WHERE f.deleted_at IS NULL AND f.file_hash <> '' AND f.file_missing = 0 AND f.file_primary = 1 AND f.file_type = 'jpg' ORDER BY p.photo_quality DESC, pl.uncertainty ASC, p.taken_at DESC LIMIT 1 ) WHERE thumb_src = '' AND deleted_at IS NULL`)). - Error; err != nil { - return err + Error + + if err == nil { + log.Debugf("previews: updated labels [%s]", time.Since(start)) + } else if strings.Contains(err.Error(), "Error 1054") { + log.Errorf("previews: failed updating labels, deprecated or unsupported database") + log.Tracef("previews: %s", err) + return nil } - log.Debugf("previews: updated labels [%s]", time.Since(start)) - - return nil + return err } // UpdateCategoryPreviews updates category preview images. @@ -132,7 +156,7 @@ func UpdateCategoryPreviews() (err error) { start := time.Now() // Categories. - if err = Db().Table(entity.Label{}.TableName()). + err = Db().Table(entity.Label{}.TableName()). UpdateColumn("thumb", gorm.Expr(`( SELECT f.file_hash FROM files f JOIN photos_labels pl ON pl.photo_id = f.photo_id AND pl.uncertainty < 100 @@ -141,13 +165,17 @@ func UpdateCategoryPreviews() (err error) { WHERE f.deleted_at IS NULL AND f.file_hash <> '' AND f.file_missing = 0 AND f.file_primary = 1 AND f.file_type = 'jpg' ORDER BY p.photo_quality DESC, pl.uncertainty ASC, p.taken_at DESC LIMIT 1 ) WHERE thumb IS NULL AND thumb_src = '' AND deleted_at IS NULL`)). - Error; err != nil { - return err + Error + + if err == nil { + log.Debugf("previews: updated categories [%s]", time.Since(start)) + } else if strings.Contains(err.Error(), "Error 1054") { + log.Errorf("previews: failed updating categories, deprecated or unsupported database") + log.Tracef("previews: %s", err) + return nil } - log.Debugf("previews: updated categories [%s]", time.Since(start)) - - return nil + return err } // UpdateSubjectPreviews updates subject preview images. @@ -191,7 +219,13 @@ func UpdateSubjectPreviews() (err error) { */ - log.Debugf("previews: updated subjects [%s]", time.Since(start)) + if err == nil { + log.Debugf("previews: updated subjects [%s]", time.Since(start)) + } else if strings.Contains(err.Error(), "Error 1054") { + log.Errorf("previews: failed updating subjects, deprecated or unsupported database") + log.Tracef("previews: %s", err) + return nil + } return err } diff --git a/internal/search/photos.go b/internal/search/photos.go index 0b8abd458..32744bf7c 100644 --- a/internal/search/photos.go +++ b/internal/search/photos.go @@ -20,7 +20,7 @@ func Photos(f form.PhotoSearch) (results PhotoResults, count int, err error) { start := time.Now() if err := f.ParseQueryString(); err != nil { - return results, 0, err + return PhotoResults{}, 0, err } s := UnscopedDb() @@ -114,15 +114,15 @@ func Photos(f form.PhotoSearch) (results PhotoResults, count int, err error) { if f.Label != "" { if err := Db().Where(AnySlug("label_slug", f.Label, txt.Or)).Or(AnySlug("custom_slug", f.Label, txt.Or)).Find(&labels).Error; len(labels) == 0 || err != nil { - log.Errorf("search: labels %s not found", txt.Quote(f.Label)) - return results, 0, fmt.Errorf("%s not found", txt.Quote(f.Label)) + log.Debugf("search: label %s not found", txt.QuoteLower(f.Label)) + return PhotoResults{}, 0, nil } else { for _, l := range labels { labelIds = append(labelIds, l.ID) Db().Where("category_id = ?", l.ID).Find(&categories) - log.Infof("search: label %s includes %d categories", txt.Quote(l.LabelName), len(categories)) + log.Infof("search: label %s includes %d categories", txt.QuoteLower(l.LabelName), len(categories)) for _, category := range categories { labelIds = append(labelIds, category.LabelID) @@ -188,7 +188,7 @@ func Photos(f form.PhotoSearch) (results PhotoResults, count int, err error) { } } else if f.Query != "" { if err := Db().Where(AnySlug("custom_slug", f.Query, " ")).Find(&labels).Error; len(labels) == 0 || err != nil { - log.Debugf("search: label %s not found, using fuzzy search", txt.Quote(f.Query)) + log.Debugf("search: label %s not found, using fuzzy search", txt.QuoteLower(f.Query)) for _, where := range LikeAnyKeyword("k.keyword", f.Query) { s = s.Where("photos.id IN (SELECT pk.photo_id FROM keywords k JOIN photos_keywords pk ON k.id = pk.keyword_id WHERE (?))", gorm.Expr(where)) @@ -199,7 +199,7 @@ func Photos(f form.PhotoSearch) (results PhotoResults, count int, err error) { Db().Where("category_id = ?", l.ID).Find(&categories) - log.Debugf("search: label %s includes %d categories", txt.Quote(l.LabelName), len(categories)) + log.Debugf("search: label %s includes %d categories", txt.QuoteLower(l.LabelName), len(categories)) for _, category := range categories { labelIds = append(labelIds, category.LabelID) diff --git a/internal/search/photos_geo.go b/internal/search/photos_geo.go index aaaaf9a21..a542fcc9e 100644 --- a/internal/search/photos_geo.go +++ b/internal/search/photos_geo.go @@ -22,7 +22,7 @@ func PhotosGeo(f form.PhotoSearchGeo) (results GeoResults, err error) { start := time.Now() if err := f.ParseQueryString(); err != nil { - return results, err + return GeoResults{}, err } s := UnscopedDb() @@ -78,7 +78,7 @@ func PhotosGeo(f form.PhotoSearchGeo) (results GeoResults, err error) { var labelIds []uint if err := Db().Where(AnySlug("custom_slug", f.Query, " ")).Find(&labels).Error; len(labels) == 0 || err != nil { - log.Debugf("search: label %s not found, using fuzzy search", txt.Quote(f.Query)) + log.Debugf("search: label %s not found, using fuzzy search", txt.QuoteLower(f.Query)) for _, where := range LikeAnyKeyword("k.keyword", f.Query) { s = s.Where("photos.id IN (SELECT pk.photo_id FROM keywords k JOIN photos_keywords pk ON k.id = pk.keyword_id WHERE (?))", gorm.Expr(where)) @@ -89,7 +89,7 @@ func PhotosGeo(f form.PhotoSearchGeo) (results GeoResults, err error) { Db().Where("category_id = ?", l.ID).Find(&categories) - log.Debugf("search: label %s includes %d categories", txt.Quote(l.LabelName), len(categories)) + log.Debugf("search: label %s includes %d categories", txt.QuoteLower(l.LabelName), len(categories)) for _, category := range categories { labelIds = append(labelIds, category.LabelID) diff --git a/internal/search/photos_test.go b/internal/search/photos_test.go index 414fa5ee8..f825a87fd 100644 --- a/internal/search/photos_test.go +++ b/internal/search/photos_test.go @@ -101,10 +101,11 @@ func TestPhotos(t *testing.T) { frm.Count = 10 frm.Offset = 0 - photos, _, err := Photos(frm) + photos, count, err := Photos(frm) - assert.Equal(t, "dog not found", err.Error()) - assert.Empty(t, photos) + assert.NoError(t, err) + assert.Equal(t, PhotoResults{}, photos) + assert.Equal(t, 0, count) }) t.Run("label query landscape", func(t *testing.T) { var frm form.PhotoSearch @@ -127,14 +128,11 @@ func TestPhotos(t *testing.T) { frm.Count = 10 frm.Offset = 0 - photos, _, err := Photos(frm) + photos, count, err := Photos(frm) - assert.Error(t, err) - assert.Empty(t, photos) - - if err != nil { - assert.Equal(t, err.Error(), "xxx not found") - } + assert.NoError(t, err) + assert.Equal(t, PhotoResults{}, photos) + assert.Equal(t, 0, count) }) t.Run("form.location true", func(t *testing.T) { var frm form.PhotoSearch diff --git a/pkg/txt/clip.go b/pkg/txt/clip.go index b1d8721b4..df7ca80b1 100644 --- a/pkg/txt/clip.go +++ b/pkg/txt/clip.go @@ -1,12 +1,17 @@ package txt -import "strings" +import ( + "strings" +) const ( - ClipDefault = 160 + Ellipsis = "…" ClipKeyword = 40 ClipSlug = 80 - ClipVarchar = 255 + ClipCategory = 100 + ClipDefault = 160 + ClipName = 160 + ClipTitle = 200 ClipQuery = 1000 ClipDescription = 16000 ) @@ -25,13 +30,20 @@ func Clip(s string, size int) string { s = string(runes[0 : size-1]) } - return s + return strings.TrimSpace(s) } -func TrimLen(s string, size int) string { - if len(s) < size || size < 4 { +// Shorten shortens a string with suffix. +func Shorten(s string, size int, suffix string) string { + if suffix == "" { + suffix = Ellipsis + } + + l := len(suffix) + + if len(s) < size || size < l+1 { return s } - return Clip(s, size-3) + "..." + return Clip(s, size-l) + suffix } diff --git a/pkg/txt/clip_test.go b/pkg/txt/clip_test.go index 74e369268..58f773da3 100644 --- a/pkg/txt/clip_test.go +++ b/pkg/txt/clip_test.go @@ -7,22 +7,32 @@ import ( ) func TestClip(t *testing.T) { - t.Run("clip", func(t *testing.T) { - assert.Equal(t, "I'm ä", Clip("I'm ä lazy BRoWN fox!", 6)) - }) - t.Run("ok", func(t *testing.T) { + t.Run("ShortEnough", func(t *testing.T) { assert.Equal(t, "I'm ä lazy BRoWN fox!", Clip("I'm ä lazy BRoWN fox!", 128)) }) - t.Run("empty", func(t *testing.T) { + t.Run("Clip", func(t *testing.T) { + assert.Equal(t, "I'm ä", Clip("I'm ä lazy BRoWN fox!", 6)) + assert.Equal(t, "I'm ä", Clip("I'm ä lazy BRoWN fox!", 7)) + }) + t.Run("TrimSpace", func(t *testing.T) { + assert.Equal(t, "abc", Clip(" abc ty3q5y4y46uy", 4)) + }) + t.Run("Empty", func(t *testing.T) { assert.Equal(t, "", Clip("", -1)) }) } -func TestTrimLen(t *testing.T) { - t.Run("len < size", func(t *testing.T) { - assert.Equal(t, "fox!", TrimLen("fox!", 6)) +func TestShorten(t *testing.T) { + t.Run("ShortEnough", func(t *testing.T) { + assert.Equal(t, "fox!", Shorten("fox!", 6, "...")) }) - t.Run("len > size", func(t *testing.T) { - assert.Equal(t, "I'm ...", TrimLen("I'm ä lazy BRoWN fox!", 8)) + t.Run("CustomSuffix", func(t *testing.T) { + assert.Equal(t, "I'm...", Shorten("I'm ä lazy BRoWN fox!", 8, "...")) + }) + t.Run("DefaultSuffix", func(t *testing.T) { + assert.Equal(t, "I'm…", Shorten("I'm ä lazy BRoWN fox!", 7, "")) + }) + t.Run("Empty", func(t *testing.T) { + assert.Equal(t, "", Shorten("", -1, "")) }) } diff --git a/pkg/txt/names.go b/pkg/txt/names.go index 620050262..d556cc9a3 100644 --- a/pkg/txt/names.go +++ b/pkg/txt/names.go @@ -3,8 +3,6 @@ package txt import ( "fmt" "strings" - - "github.com/gosimple/slug" ) // UniqueNames removes exact duplicates from a list of strings without changing their order. @@ -110,22 +108,12 @@ func NormalizeName(name string) string { return r }, name) - // Shorten. - name = Clip(name, ClipDefault) + name = strings.TrimSpace(name) if name == "" { return "" } - // Capitalize. - return Title(name) -} - -// NameSlug converts a name to a valid slug. -func NameSlug(name string) string { - if name == "" { - return "" - } - - return slug.Make(Clip(name, ClipSlug)) + // Shorten and capitalize. + return Clip(Title(name), ClipDefault) } diff --git a/pkg/txt/names_test.go b/pkg/txt/names_test.go index 1d7693750..e84da7794 100644 --- a/pkg/txt/names_test.go +++ b/pkg/txt/names_test.go @@ -129,18 +129,3 @@ func TestNormalizeName(t *testing.T) { assert.Equal(t, "陈 赵", NormalizeName(" 陈 赵")) }) } - -func TestNameSlug(t *testing.T) { - t.Run("Empty", func(t *testing.T) { - assert.Equal(t, "", NameSlug("")) - }) - t.Run("BillGates", func(t *testing.T) { - assert.Equal(t, "william-henry-gates-iii", NameSlug("William Henry Gates III")) - }) - t.Run("Quotes", func(t *testing.T) { - assert.Equal(t, "william-henry-gates", NameSlug("william \"HenRy\" gates' ")) - }) - t.Run("Chinese", func(t *testing.T) { - assert.Equal(t, "chen-zhao", NameSlug(" 陈 赵")) - }) -} diff --git a/pkg/txt/quote.go b/pkg/txt/quote.go index dcdccfe98..fd2a2a254 100644 --- a/pkg/txt/quote.go +++ b/pkg/txt/quote.go @@ -13,3 +13,8 @@ func Quote(text string) string { return text } + +// QuoteLower converts a string to lowercase and adds quotation marks if needed. +func QuoteLower(text string) string { + return Quote(strings.ToLower(text)) +} diff --git a/pkg/txt/quote_test.go b/pkg/txt/quote_test.go index 87cd1e596..277bd5be1 100644 --- a/pkg/txt/quote_test.go +++ b/pkg/txt/quote_test.go @@ -17,3 +17,15 @@ func TestQuote(t *testing.T) { assert.Equal(t, "“”", Quote("")) }) } + +func TestQuoteLower(t *testing.T) { + t.Run("The quick brown fox.", func(t *testing.T) { + assert.Equal(t, "“the quick brown fox.”", QuoteLower("The quick brown fox.")) + }) + t.Run("filename.txt", func(t *testing.T) { + assert.Equal(t, "filename.txt", QuoteLower("filename.txt")) + }) + t.Run("empty string", func(t *testing.T) { + assert.Equal(t, "“”", QuoteLower("")) + }) +} diff --git a/pkg/txt/slug.go b/pkg/txt/slug.go index 4dd7b796c..4849ed0ac 100644 --- a/pkg/txt/slug.go +++ b/pkg/txt/slug.go @@ -1,6 +1,21 @@ package txt -import "strings" +import ( + "strings" + + "github.com/gosimple/slug" +) + +// Slug converts a string to a valid slug with a max length of 80 runes. +func Slug(s string) string { + s = strings.TrimSpace(s) + + if s == "" { + return "" + } + + return Clip(slug.Make(s), ClipSlug) +} // SlugToTitle converts a slug back to a title func SlugToTitle(s string) string { diff --git a/pkg/txt/slug_test.go b/pkg/txt/slug_test.go index 1baa1a9c9..0d807200f 100644 --- a/pkg/txt/slug_test.go +++ b/pkg/txt/slug_test.go @@ -6,6 +6,21 @@ import ( "github.com/stretchr/testify/assert" ) +func TestSlug(t *testing.T) { + t.Run("Empty", func(t *testing.T) { + assert.Equal(t, "", Slug("")) + }) + t.Run("BillGates", func(t *testing.T) { + assert.Equal(t, "william-henry-gates-iii", Slug("William Henry Gates III")) + }) + t.Run("Quotes", func(t *testing.T) { + assert.Equal(t, "william-henry-gates", Slug("william \"HenRy\" gates' ")) + }) + t.Run("Chinese", func(t *testing.T) { + assert.Equal(t, "chen-zhao", Slug(" 陈 赵")) + }) +} + func TestSlugToTitle(t *testing.T) { t.Run("cute_Kitten", func(t *testing.T) { assert.Equal(t, "Cute-Kitten", SlugToTitle("cute-kitten")) diff --git a/scripts/sql/init-test-databases.sql b/scripts/sql/init-test-databases.sql index d4406e272..22c291cc0 100644 --- a/scripts/sql/init-test-databases.sql +++ b/scripts/sql/init-test-databases.sql @@ -1,8 +1,8 @@ CREATE DATABASE IF NOT EXISTS alpha; CREATE DATABASE IF NOT EXISTS beta; CREATE DATABASE IF NOT EXISTS gamma; -CREATE DATABASE IF NOT EXISTS delta; -CREATE DATABASE IF NOT EXISTS epsilon; +CREATE DATABASE IF NOT EXISTS latest; +CREATE DATABASE IF NOT EXISTS preview; DROP DATABASE IF EXISTS acceptance; CREATE DATABASE IF NOT EXISTS acceptance; DROP DATABASE IF EXISTS api;