Videos: Refine still image extraction with ffmpeg #5189

Signed-off-by: Michael Mayer <michael@photoprism.app>

internal/ffmpeg/encode/offset.go

@@ -2,11 +2,35 @@ package encode
 
 import "time"
 
-// PreviewTimeOffset returns a time offset depending on the video duration for extracting a cover image,
+// PreviewSeekOffset returns a seek offset depending on the video duration for extracting a preview image,
+// see https://trac.ffmpeg.org/wiki/Seeking and https://ffmpeg.org/ffmpeg-utils.html#time-duration-syntax.
+func PreviewSeekOffset(d time.Duration) string {
+	// Default time offset.
+	result := "00:00:00.000"
+
+	if d <= 0 {
+		return result
+	}
+
+	// If the video is long enough, don't use the first frames to avoid completely
+	// black or white thumbnails in case there is an effect or intro.
+	switch {
+	case d > time.Hour:
+		result = "00:02:28.000"
+	case d > 10*time.Minute:
+		result = "00:00:58.000"
+	case d > 3*time.Minute:
+		result = "00:00:28.000"
+	}
+
+	return result
+}
+
+// PreviewTimeOffset returns a time offset depending on the video duration for extracting a preview image,
 // see https://trac.ffmpeg.org/wiki/Seeking and https://ffmpeg.org/ffmpeg-utils.html#time-duration-syntax.
 func PreviewTimeOffset(d time.Duration) string {
 	// Default time offset.
-	result := "00:00:00.000"
+	result := "00:00:00.001"
 
 	if d <= 0 {
 		return result

internal/ffmpeg/encode/offset_test.go

@@ -7,9 +7,33 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestPreviewSeekOffset(t *testing.T) {
+	t.Run("Second", func(t *testing.T) {
+		assert.Equal(t, "00:00:00.000", PreviewSeekOffset(time.Second))
+	})
+	t.Run("Minute", func(t *testing.T) {
+		assert.Equal(t, "00:00:00.000", PreviewSeekOffset(time.Minute+1))
+	})
+	t.Run("FiveMinutes", func(t *testing.T) {
+		assert.Equal(t, "00:00:28.000", PreviewSeekOffset(5*time.Minute))
+	})
+	t.Run("FifteenMinutes", func(t *testing.T) {
+		assert.Equal(t, "00:00:58.000", PreviewSeekOffset(15*time.Minute))
+	})
+	t.Run("HalfHour", func(t *testing.T) {
+		assert.Equal(t, "00:00:58.000", PreviewSeekOffset(30*time.Minute))
+	})
+	t.Run("Hour", func(t *testing.T) {
+		assert.Equal(t, "00:00:58.000", PreviewSeekOffset(time.Hour))
+	})
+	t.Run("ThreeHours", func(t *testing.T) {
+		assert.Equal(t, "00:02:28.000", PreviewSeekOffset(3*time.Hour))
+	})
+}
+
 func TestPreviewTimeOffset(t *testing.T) {
 	t.Run("Second", func(t *testing.T) {
-		assert.Equal(t, "00:00:00.000", PreviewTimeOffset(time.Second))
+		assert.Equal(t, "00:00:00.001", PreviewTimeOffset(time.Second))
 	})
 	t.Run("Minute", func(t *testing.T) {
 		assert.Equal(t, "00:00:09.000", PreviewTimeOffset(time.Minute+1))

internal/ffmpeg/encode/options.go

@@ -19,6 +19,7 @@ type Options struct {
 	MapVideo    string        // See https://trac.ffmpeg.org/wiki/Map#Videostreamsonly
 	MapAudio    string        // See https://trac.ffmpeg.org/wiki/Map#Audiostreamsonly
 	MapMetadata string        // See https://ffmpeg.org/ffmpeg.html
+	SeekOffset  string        // See https://trac.ffmpeg.org/wiki/Seeking and https://ffmpeg.org/ffmpeg-utils.html#time-duration-syntax
 	TimeOffset  string        // See https://trac.ffmpeg.org/wiki/Seeking and https://ffmpeg.org/ffmpeg-utils.html#time-duration-syntax
 	Duration    time.Duration // See https://ffmpeg.org/ffmpeg.html#Main-options
 	MovFlags    string
@@ -105,8 +106,12 @@ func NewRemuxOptions(ffmpegBin string, container fs.Type, force bool) Options {
 // NewPreviewImageOptions generates encoding options for extracting a video preview image.
 func NewPreviewImageOptions(ffmpegBin string, videoDuration time.Duration) *Options {
 	return &Options{
-		Bin:        ffmpegBin,
+		Bin:         ffmpegBin,
-		TimeOffset: PreviewTimeOffset(videoDuration),
+		MapVideo:    DefaultMapVideo,
+		MapAudio:    DefaultMapAudio,
+		MapMetadata: DefaultMapMetadata,
+		SeekOffset:  PreviewSeekOffset(videoDuration),
+		TimeOffset:  PreviewTimeOffset(videoDuration),
 	}
 }
 

internal/ffmpeg/extract_image_cmd.go

@@ -28,11 +28,19 @@ func ExtractJpegImageCmd(videoName, imageName string, opt *encode.Options) *exec
 	// "-vf", "zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=tonemap=gamma:desat=0,zscale=t=bt709:m=bt709:r=tv,format=yuv420p",
 	return exec.Command(
 		opt.Bin,
-		"-hide_banner", "-y", "-strict", "-2", "-loglevel", "error",
+		"-hide_banner",
-		"-ss", opt.TimeOffset, "-i", videoName,
+		"-loglevel", "error",
-		"-skip_frame", "nokey", // skip non-keyframes
+		"-y", "-strict", "-2", // support new video codecs
-		"-frames:v", "1",
+		"-hwaccel", "none", // disable hardware acceleration
-		imageName,
+		"-err_detect", "ignore_err", // ignore errors
+		"-ss", opt.SeekOffset, // open video at this position
+		"-i", videoName, // input video file name
+		"-ss", opt.TimeOffset, // extract image at this position
+		// "-map", opt.MapVideo, "-an", "-sn", "-dn", // map streams (seems not required)
+		// "-skip_frame", "nokey", // skip non-keyframes
+		"-vf", "setparams=range=tv:color_primaries=bt709:color_trc=bt709:colorspace=bt709,scale=trunc(iw/2)*2:trunc(ih/2)*2,setsar=1,format=yuvj422p",
+		"-frames:v", "1", // extract one frame
+		imageName, // output image file name
 	)
 }
 
@@ -40,10 +48,18 @@ func ExtractJpegImageCmd(videoName, imageName string, opt *encode.Options) *exec
 func ExtractPngImageCmd(videoName, imageName string, opt *encode.Options) *exec.Cmd {
 	return exec.Command(
 		opt.Bin,
-		"-hide_banner", "-y", "-strict", "-2", "-loglevel", "error",
+		"-hide_banner",
-		"-ss", opt.TimeOffset, "-i", videoName,
+		"-loglevel", "error",
-		"-skip_frame", "nokey", // skip non-keyframes
+		"-y", "-strict", "-2", // support new video codecs
-		"-frames:v", "1",
+		"-hwaccel", "none", // disable hardware acceleration
-		imageName,
+		"-err_detect", "ignore_err", // ignore errors
+		"-ss", opt.SeekOffset, // open video at this position
+		"-i", videoName, // input video file name
+		"-ss", opt.TimeOffset, // extract image at this position
+		// "-map", opt.MapVideo, "-an", "-sn", "-dn", // map streams (seems not required)
+		// "-skip_frame", "nokey", // skip non-keyframes
+		"-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2,setsar=1",
+		"-frames:v", "1", // extract one frame
+		imageName, // output image file name
 	)
 }

internal/ffmpeg/extract_image_cmd_test.go

@@ -23,7 +23,7 @@ func TestExtractImageCmd(t *testing.T) {
 	cmdStr = strings.Replace(cmdStr, srcName, "SRC", 1)
 	cmdStr = strings.Replace(cmdStr, destName, "DEST", 1)
 
-	assert.Equal(t, "/usr/bin/ffmpeg -hide_banner -y -strict -2 -loglevel error -ss 00:00:00.000 -i SRC -skip_frame nokey -frames:v 1 DEST", cmdStr)
+	assert.Equal(t, "/usr/bin/ffmpeg -hide_banner -loglevel error -y -strict -2 -hwaccel none -err_detect ignore_err -ss 00:00:00.000 -i SRC -ss 00:00:00.001 -vf setparams=range=tv:color_primaries=bt709:color_trc=bt709:colorspace=bt709,scale=trunc(iw/2)*2:trunc(ih/2)*2,setsar=1,format=yuvj422p -frames:v 1 DEST", cmdStr)
 
 	RunCommandTest(t, "jpg", srcName, destName, cmd, true)
 }
@@ -40,7 +40,7 @@ func TestExtractJpegImageCmd(t *testing.T) {
 	cmdStr = strings.Replace(cmdStr, srcName, "SRC", 1)
 	cmdStr = strings.Replace(cmdStr, destName, "DEST", 1)
 
-	assert.Equal(t, "/usr/bin/ffmpeg -hide_banner -y -strict -2 -loglevel error -ss 00:00:00.000 -i SRC -skip_frame nokey -frames:v 1 DEST", cmdStr)
+	assert.Equal(t, "/usr/bin/ffmpeg -hide_banner -loglevel error -y -strict -2 -hwaccel none -err_detect ignore_err -ss 00:00:00.000 -i SRC -ss 00:00:00.001 -vf setparams=range=tv:color_primaries=bt709:color_trc=bt709:colorspace=bt709,scale=trunc(iw/2)*2:trunc(ih/2)*2,setsar=1,format=yuvj422p -frames:v 1 DEST", cmdStr)
 
 	RunCommandTest(t, "jpeg", srcName, destName, cmd, true)
 }
@@ -57,7 +57,7 @@ func TestExtractPngImageCmd(t *testing.T) {
 	cmdStr = strings.Replace(cmdStr, srcName, "SRC", 1)
 	cmdStr = strings.Replace(cmdStr, destName, "DEST", 1)
 
-	assert.Equal(t, "/usr/bin/ffmpeg -hide_banner -y -strict -2 -loglevel error -ss 00:00:00.000 -i SRC -skip_frame nokey -frames:v 1 DEST", cmdStr)
+	assert.Equal(t, "/usr/bin/ffmpeg -hide_banner -loglevel error -y -strict -2 -hwaccel none -err_detect ignore_err -ss 00:00:00.000 -i SRC -ss 00:00:00.001 -vf scale=trunc(iw/2)*2:trunc(ih/2)*2,setsar=1 -frames:v 1 DEST", cmdStr)
 
 	RunCommandTest(t, "png", srcName, destName, cmd, true)
 }