chunksize: initial implementation of chunksize helper lib

fs/chunksize/chunksize.go

@@ -0,0 +1,36 @@
+// Package chunksize calculates a suitable chunk size for large uploads
+package chunksize
+
+import (
+	"github.com/rclone/rclone/fs"
+)
+
+/*
+ Calculator calculates the minimum chunk size needed to fit within the maximum number of parts, rounded up to the nearest fs.Mebi
+
+ For most backends, (chunk_size) * (concurrent_upload_routines) memory will be required so we want to use the smallest
+ possible chunk size that's going to allow the upload to proceed. Rounding up to the nearest fs.Mebi on the assumption
+ that some backends may only allow integer type parameters when specifying the chunk size.
+
+ Returns the default chunk size if it is sufficiently large enough to support the given file size otherwise returns the
+ smallest chunk size necessary to allow the upload to proceed.
+*/
+func Calculator(objInfo fs.ObjectInfo, maxParts int, defaultChunkSize fs.SizeSuffix) fs.SizeSuffix {
+	fileSize := fs.SizeSuffix(objInfo.Size())
+	requiredChunks := fileSize / defaultChunkSize
+	if requiredChunks < fs.SizeSuffix(maxParts) || (requiredChunks == fs.SizeSuffix(maxParts) && fileSize%defaultChunkSize == 0) {
+		return defaultChunkSize
+	}
+
+	minChunk := fileSize / fs.SizeSuffix(maxParts)
+	remainder := minChunk % fs.Mebi
+	if remainder != 0 {
+		minChunk += fs.Mebi - remainder
+	}
+	if fileSize/minChunk == fs.SizeSuffix(maxParts) && fileSize%fs.SizeSuffix(maxParts) != 0 { // when right on the boundary, we need to add a MiB
+		minChunk += fs.Mebi
+	}
+
+	fs.Debugf(objInfo, "size: %v, parts: %v, default: %v, new: %v; default chunk size insufficient, returned new chunk size", fileSize, maxParts, defaultChunkSize, minChunk)
+	return minChunk
+}