Videos: Improve indexing and searching #312

Signed-off-by: Michael Mayer <michael@liquidbytes.net>

frontend/src/common/viewer.js

@@ -56,6 +56,11 @@ class Viewer {
             getImageURLForShare: function (button) {
                 const item = gallery.currItem;
 
+                if (!item.original_w) {
+                    button.label = button.template.replace("size", "not available");
+                    return item.download_url;
+                }
+
                 if(button.id === "original") {
                     button.label = button.template.replace("size", item.original_w + " × " + item.original_h);
                     return item.download_url;

frontend/src/dialog/p-video-dialog.vue

@@ -32,7 +32,7 @@
                 let uri = video.videoUri();
 
                 if (!uri) {
-                    this.$notify.error("no video file found");
+                    this.$notify.error("no video selected");
                     return;
                 }
 
@@ -44,7 +44,7 @@
 
                 if(file.FileWidth > 0) {
                     width = file.FileWidth;
-                } else if(main.FileWidth > 0) {
+                } else if(main && main.FileWidth > 0) {
                     width = main.FileWidth;
                 } else {
                     width = this.defaultWidth;
@@ -52,7 +52,7 @@
 
                 if(file.FileHeight > 0) {
                     height = file.FileHeight;
-                } else if(main.FileHeight > 0) {
+                } else if(main && main.FileHeight > 0) {
                     height = main.FileHeight;
                 } else {
                     height = this.defaultHeight;

frontend/src/model/photo.js

@@ -178,6 +178,12 @@ class Photo extends RestModel {
         let hash = this.mainFileHash();
 
         if (!hash) {
+            let video = this.videoFile();
+
+            if (video && video.FileHash) {
+                return "/api/v1/thumbnails/" + video.FileHash + "/" + type;
+            }
+
             return "/api/v1/svg/photo";
         }
 
@@ -278,6 +284,11 @@ class Photo extends RestModel {
 
         if (file.FileWidth && file.FileHeight) {
             info.push(file.FileWidth + " × " + file.FileHeight);
+        } else if (!file.FilePrimary) {
+            let main = this.mainFile();
+            if (main && main.FileWidth && main.FileHeight) {
+                info.push(main.FileWidth + " × " + main.FileHeight);
+            }
         }
 
         if (file.FileSize > 102400) {

frontend/src/model/thumb.js

@@ -9,8 +9,8 @@ class Thumb extends Model {
             uuid: "",
             title: "",
             favorite: false,
-            original_w: "",
+            original_w: 0,
-            original_h: "",
+            original_h: 0,
             download_url: "",
         };
     }
@@ -25,15 +25,33 @@ class Thumb extends Model {
         }
     }
 
+    static thumbNotFound() {
+        const result = {
+            uuid: "",
+            title: "Not Found",
+            favorite: false,
+            original_w: 0,
+            original_h: 0,
+            download_url: "",
+        };
+
+        for (let i = 0; i < thumbs.length; i++) {
+            result[thumbs[i].Name] = {
+                src: "/api/v1/svg/photo",
+                w: thumbs[i].Width,
+                h: thumbs[i].Height,
+            };
+        }
+
+        return result;
+    }
+
     static fromPhotos(photos) {
         let result = [];
 
         photos.forEach((p) => {
             let thumb = this.fromPhoto(p);
-
-            if(thumb) {
             result.push(thumb);
-            }
         });
 
         return result;
@@ -45,7 +63,7 @@ class Thumb extends Model {
         }
 
         if (!photo || !photo.FileHash) {
-            return false;
+            return this.thumbNotFound();
         }
 
         const result = {
@@ -72,7 +90,7 @@ class Thumb extends Model {
 
     static fromFile(photo, file) {
         if (!photo || !file || !file.FileHash) {
-            return false;
+            return this.thumbNotFound();
         }
 
         const result = {

go.mod

@@ -11,8 +11,8 @@ require (
 	github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 // indirect
 	github.com/disintegration/imaging v1.6.2
 	github.com/djherbis/times v1.2.0
-	github.com/dsoprea/go-exif/v2 v2.0.0-20200506085928-e7aea1340ccb
+	github.com/dsoprea/go-exif/v2 v2.0.0-20200516122116-a45cc7cfd55e
-	github.com/dsoprea/go-jpeg-image-structure v0.0.0-20200504080818-288e72e0addf
+	github.com/dsoprea/go-jpeg-image-structure v0.0.0-20200516131022-2c0537684b05
 	github.com/dsoprea/go-png-image-structure v0.0.0-20200402000326-c0fdb803026f
 	github.com/dsoprea/go-utility v0.0.0-20200512094054-1abbbc781176 // indirect
 	github.com/dustin/go-humanize v1.0.0

go.sum

@@ -70,10 +70,14 @@ github.com/dsoprea/go-exif/v2 v2.0.0-20200506054703-8da3881353b3 h1:hgFwhM9Sjozd
 github.com/dsoprea/go-exif/v2 v2.0.0-20200506054703-8da3881353b3/go.mod h1:YXOyDqCYjBuHHRw4JIGPgOgMit0IDvVSjjhsqOAFTYQ=
 github.com/dsoprea/go-exif/v2 v2.0.0-20200506085928-e7aea1340ccb h1:En5n3RCyOL1Wnf78SNCzqFT9u4+kWJoSQEK/Z8MoZQE=
 github.com/dsoprea/go-exif/v2 v2.0.0-20200506085928-e7aea1340ccb/go.mod h1:YXOyDqCYjBuHHRw4JIGPgOgMit0IDvVSjjhsqOAFTYQ=
+github.com/dsoprea/go-exif/v2 v2.0.0-20200516122116-a45cc7cfd55e h1:tPHXVRs63sg0ajoZjdmMa5aZuyjnSAt3Anwh2F4XsJM=
+github.com/dsoprea/go-exif/v2 v2.0.0-20200516122116-a45cc7cfd55e/go.mod h1:YXOyDqCYjBuHHRw4JIGPgOgMit0IDvVSjjhsqOAFTYQ=
 github.com/dsoprea/go-jpeg-image-structure v0.0.0-20200419165912-75b7a4f392e6 h1:6oyE0L+MX1iUjldwrLdAaU95g36UrKpbmlyslhyoJj4=
 github.com/dsoprea/go-jpeg-image-structure v0.0.0-20200419165912-75b7a4f392e6/go.mod h1:5vwlfhyZI7u8AuvTl0G70sdqVdH41f7dscvBQ6mEbHs=
 github.com/dsoprea/go-jpeg-image-structure v0.0.0-20200504080818-288e72e0addf h1:fHaBuk2M/HNmLyQlQH91UNmsc883sNLukYXEsk7coZM=
 github.com/dsoprea/go-jpeg-image-structure v0.0.0-20200504080818-288e72e0addf/go.mod h1:dSPqu4ZEK+hLQmKh1XkwVdNkWwI8G7TOiZMLZrxx/j0=
+github.com/dsoprea/go-jpeg-image-structure v0.0.0-20200516131022-2c0537684b05 h1:aBQuseJ46TDgCd1a/hYzJvvWNRTbGwt6k8/lOYzR9xU=
+github.com/dsoprea/go-jpeg-image-structure v0.0.0-20200516131022-2c0537684b05/go.mod h1:7V7bSB5y4Zwomfq6jz43dABqCs9RFCgV+gakP7lIcHY=
 github.com/dsoprea/go-logging v0.0.0-20190624164917-c4f10aab7696 h1:VGFnZAcLwPpt1sHlAxml+pGLZz9A2s+K/s1YNhPC91Y=
 github.com/dsoprea/go-logging v0.0.0-20190624164917-c4f10aab7696/go.mod h1:Nm/x2ZUNRW6Fe5C3LxdY1PyZY5wmDv/s5dkPJ/VB3iA=
 github.com/dsoprea/go-logging v0.0.0-20200502201358-170ff607885f h1:FonKAuW3PmNtqk9tOR+Z7bnyQHytmnZBCmm5z1PQMss=

internal/api/photo_thumbnail.go

@@ -37,6 +37,15 @@ func GetThumbnail(router *gin.RouterGroup, conf *config.Config) {
 			return
 		}
 
+		if f.FileVideo {
+			f, err = query.FileByPhotoUUID(f.PhotoUUID)
+
+			if err != nil {
+				c.Data(http.StatusOK, "image/svg+xml", videoIconSvg)
+				return
+			}
+		}
+
 		if f.FileError != "" {
 			c.Data(http.StatusOK, "image/svg+xml", brokenIconSvg)
 			return

internal/api/svg.go

@@ -7,10 +7,8 @@ import (
 )
 
 var photoIconSvg = []byte(`
-<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24">
+<svg xmlns="http://www.w3.org/2000/svg" height="24" viewBox="0 0 24 24" width="24"><path d="M0 0h24v24H0V0z" fill="none"/>
-<path d="M0 0h24v24H0z" fill="none"/>
+<path d="M19 5v14H5V5h14m0-2H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zm-4.86 8.86l-3 3.87L9 13.14 6 17h12l-3.86-5.14z"/></svg>`)
-<path d="M21 19V5c0-1.1-.9-2-2-2H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2zM8.5 13.5l2.5 3.01L14.5 12l4.5 6H5l3.5-4.5z"/>
-</svg>`)
 
 var videoIconSvg = []byte(`<svg xmlns="http://www.w3.org/2000/svg" height="24" viewBox="0 0 24 24" width="24">
 <path d="M0 0h24v24H0z" fill="none"/><path d="M10 8v8l5-4-5-4zm9-5H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zm0 16H5V5h14v14z"/></svg>`)

internal/entity/file.go

@@ -8,6 +8,7 @@ import (
 	"github.com/gosimple/slug"
 	"github.com/jinzhu/gorm"
 	"github.com/photoprism/photoprism/pkg/rnd"
+	"github.com/ulule/deepcopier"
 )
 
 // File represents an image or sidecar file that belongs to a photo
@@ -53,6 +54,18 @@ type File struct {
 	DeletedAt       *time.Time `sql:"index"`
 }
 
+type FileInfos struct {
+	FileWidth       int
+	FileHeight      int
+	FileOrientation int
+	FileAspectRatio float32
+	FileMainColor   string
+	FileColors      string
+	FileLuminance   string
+	FileDiff        uint32
+	FileChroma      uint8
+}
+
 // FirstFileByHash gets a file in db from its hash
 func FirstFileByHash(fileHash string) (File, error) {
 	var file File
@@ -136,3 +149,14 @@ func (m *File) Save() error {
 
 	return Db().Model(m).Related(Photo{}).Error
 }
+
+// UpdateVideoInfos updates related video infos based on this file.
+func (m *File) UpdateVideoInfos() error {
+	values := FileInfos{}
+
+	if err := deepcopier.Copy(&values).From(m); err != nil {
+		return err
+	}
+
+	return Db().Model(File{}).Where("photo_id = ? AND file_video = 1", m.PhotoID).Updates(values).Error
+}

internal/meta/exif.go

@@ -150,7 +150,7 @@ func (data *Data) Exif(fileName string) (err error) {
 		return nil
 	}
 
-	_, err = exif.Visit(exifcommon.IfdStandard, im, ti, rawExif, visitor)
+	_, _, err = exif.Visit(exifcommon.IfdStandard, im, ti, rawExif, visitor)
 
 	if err != nil {
 		return err

internal/photoprism/index_mediafile.go

@@ -135,22 +135,6 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
 	photo.PhotoPath = filePath
 	photo.PhotoName = fileBase
 
-	if m.IsVideo() {
-		photo.PhotoVideo = true
-		metaData, _ = m.MetaData()
-
-		file.FileCodec = metaData.Codec
-		file.FileWidth = metaData.Width
-		file.FileHeight = metaData.Height
-		file.FileDuration = metaData.Duration
-		file.FileAspectRatio = metaData.AspectRatio()
-		file.FilePortrait = metaData.Portrait()
-
-		if res := metaData.Megapixels(); res > photo.PhotoResolution {
-			photo.PhotoResolution = res
-		}
-	}
-
 	if !file.FilePrimary {
 		if photoExists {
 			if q := ind.db.Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); q.Error != nil {
@@ -170,6 +154,35 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
 		return result
 	}
 
+	if m.IsVideo() {
+		photo.PhotoVideo = true
+		metaData, _ = m.MetaData()
+
+		file.FileCodec = metaData.Codec
+		file.FileWidth = metaData.Width
+		file.FileHeight = metaData.Height
+		file.FileDuration = metaData.Duration
+		file.FileAspectRatio = metaData.AspectRatio()
+		file.FilePortrait = metaData.Portrait()
+
+		if res := metaData.Megapixels(); res > photo.PhotoResolution {
+			photo.PhotoResolution = res
+		}
+
+		if file.FileWidth == 0 && primaryFile.FileWidth > 0 {
+			file.FileWidth = primaryFile.FileWidth
+			file.FileHeight = primaryFile.FileHeight
+			file.FileAspectRatio = primaryFile.FileAspectRatio
+			file.FilePortrait = primaryFile.FilePortrait
+		}
+
+		file.FileDiff = primaryFile.FileDiff
+		file.FileMainColor = primaryFile.FileMainColor
+		file.FileChroma = primaryFile.FileChroma
+		file.FileLuminance = primaryFile.FileLuminance
+		file.FileColors = primaryFile.FileColors
+	}
+
 	// file obviously exists: remove deleted and missing flags
 	file.DeletedAt = nil
 	file.FileMissing = false
@@ -448,6 +461,12 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
 		result.Status = IndexAdded
 	}
 
+	if photo.PhotoVideo && file.FilePrimary {
+		if err := file.UpdateVideoInfos(); err != nil {
+			log.Errorf("index: %s", err)
+		}
+	}
+
 	result.FileID = file.ID
 	result.FileUUID = file.FileUUID
 

internal/query/photos.go

@@ -262,6 +262,7 @@ func Photos(f form.PhotoSearch) (results PhotosResults, count int, err error) {
 	case entity.SortOrderImported:
 		s = s.Order("photos.id DESC, files.file_primary DESC")
 	case entity.SortOrderSimilar:
+		s = s.Where("files.file_diff > 0")
 		s = s.Order("files.file_main_color, photos.location_id, files.file_diff, taken_at DESC, files.file_primary DESC")
 	default:
 		s = s.Order("taken_at DESC, photos.photo_uuid, files.file_primary DESC")