johanels/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-12-11 22:14:05 +01:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

lsjson: Add --hash-type parameter and use it in lsf to speed up hashing

Browse Source 
Before this change if you specified --hash MD5 in rclone lsf it would
calculate all the hashes and just return the MD5 hash which was very
slow on the local backend.

Likewise specifying --hash on rclone lsjson was equally slow.

This change introduces the --hash-type flag (and corresponding
internal parameter) so that the hashes required can be selected in
lsjson.

This is used internally in lsf when the --hash parameter is selected
to speed up the hashing by only hashing with the one hash specified.

Fixes #4181
This commit is contained in:
Nick Craig-Wood
2020-04-28 18:44:04 +01:00
parent a3f0992a22
commit f37af9afec
3 changed files with 41 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
fs/operations/lsjson.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/pkg/errors"
"github.com/rclone/rclone/backend/crypt"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/walk"
)
@@ -69,14 +70,15 @@ func formatForPrecision(precision time.Duration) string {
// ListJSONOpt describes the options for ListJSON
type ListJSONOpt struct {
Recurse bool `json:"recurse"`
NoModTime bool `json:"noModTime"`
NoMimeType bool `json:"noMimeType"`
ShowEncrypted bool `json:"showEncrypted"`
ShowOrigIDs bool `json:"showOrigIDs"`
ShowHash bool `json:"showHash"`
DirsOnly bool `json:"dirsOnly"`
FilesOnly bool `json:"filesOnly"`
Recurse bool `json:"recurse"`
NoModTime bool `json:"noModTime"`
NoMimeType bool `json:"noMimeType"`
ShowEncrypted bool `json:"showEncrypted"`
ShowOrigIDs bool `json:"showOrigIDs"`
ShowHash bool `json:"showHash"`
DirsOnly bool `json:"dirsOnly"`
FilesOnly bool `json:"filesOnly"`
HashTypes []string `json:"hashTypes"` // hash types to show if ShowHash is set, eg "MD5", "SHA-1"
}
// ListJSON lists fsrc using the options in opt calling callback for each item
@@ -99,6 +101,20 @@ func ListJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt,
canGetTier := features.GetTier
format := formatForPrecision(fsrc.Precision())
isBucket := features.BucketBased && remote == "" && fsrc.Root() == "" // if bucket based remote listing the root mark directories as buckets
showHash := opt.ShowHash
hashTypes := fsrc.Hashes().Array()
if len(opt.HashTypes) != 0 {
showHash = true
hashTypes = []hash.Type{}
for _, hashType := range opt.HashTypes {
var ht hash.Type
err := ht.Set(hashType)
if err != nil {
return err
}
hashTypes = append(hashTypes, ht)
}
}
err := walk.ListR(ctx, fsrc, remote, false, ConfigMaxDepth(opt.Recurse), walk.ListAll, func(entries fs.DirEntries) (err error) {
for _, entry := range entries {
switch entry.(type) {
@@ -150,9 +166,9 @@ func ListJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt,
item.IsBucket = isBucket
case fs.Object:
item.IsDir = false
if opt.ShowHash {
if showHash {
item.Hashes = make(map[string]string)
for _, hashType := range x.Fs().Hashes().Array() {
for _, hashType := range hashTypes {
hash, err := x.Hash(ctx, hashType)
if err != nil {
fs.Errorf(x, "Failed to read hash: %v", err)