From 16971ab6b9c3bdc9fbe42df998c2e5962207a2ab Mon Sep 17 00:00:00 2001 From: hunshcn Date: Wed, 12 Nov 2025 23:15:13 +0800 Subject: [PATCH] s3: add --s3-use-data-integrity-protections to fix BadDigest error in Alibaba, Tencent Since aws/aws-sdk-go-v2#2960, aws-go-sdk-v2 changes its default integrity behavior. This breaks some s3 providers (eg Tencent, Alibaba) https://github.com/aws/aws-sdk-go-v2/discussions/2960 This introduces `use_data_integrity_protections` option to disable it. Defaults to false with it set to true for AWS. Fixes #8432 Fixes #8483 --- backend/s3/provider/AWS.yaml | 1 + backend/s3/providers.go | 29 ++++---- backend/s3/s3.go | 140 +++++++++++++++++++---------------- 3 files changed, 93 insertions(+), 77 deletions(-) diff --git a/backend/s3/provider/AWS.yaml b/backend/s3/provider/AWS.yaml index 39c12ae08..989462d32 100644 --- a/backend/s3/provider/AWS.yaml +++ b/backend/s3/provider/AWS.yaml @@ -137,3 +137,4 @@ use_accelerate_endpoint: true quirks: might_gzip: false # Never auto gzips objects use_unsigned_payload: false # AWS has trailer support which means it adds checksums in the trailer without seeking + use_data_integrity_protections: true diff --git a/backend/s3/providers.go b/backend/s3/providers.go index 6c9eb8a17..858aaea6e 100644 --- a/backend/s3/providers.go +++ b/backend/s3/providers.go @@ -20,20 +20,21 @@ var NewYamlMap = orderedmap.New[string, string] // Quirks defines all the S3 provider quirks type Quirks struct { - ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2 - ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style - ListURLEncode *bool `yaml:"list_url_encode,omitempty"` - UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"` - UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"` - UseAcceptEncodingGzip *bool `yaml:"use_accept_encoding_gzip,omitempty"` - MightGzip *bool `yaml:"might_gzip,omitempty"` - UseMultipartUploads *bool `yaml:"use_multipart_uploads,omitempty"` - UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"` - UseXID *bool `yaml:"use_x_id,omitempty"` - SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"` - CopyCutoff *int64 `yaml:"copy_cutoff,omitempty"` - MaxUploadParts *int `yaml:"max_upload_parts,omitempty"` - MinChunkSize *int64 `yaml:"min_chunk_size,omitempty"` + ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2 + ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style + ListURLEncode *bool `yaml:"list_url_encode,omitempty"` + UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"` + UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"` + UseAcceptEncodingGzip *bool `yaml:"use_accept_encoding_gzip,omitempty"` + UseDataIntegrityProtections *bool `yaml:"use_data_integrity_protections,omitempty"` + MightGzip *bool `yaml:"might_gzip,omitempty"` + UseMultipartUploads *bool `yaml:"use_multipart_uploads,omitempty"` + UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"` + UseXID *bool `yaml:"use_x_id,omitempty"` + SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"` + CopyCutoff *int64 `yaml:"copy_cutoff,omitempty"` + MaxUploadParts *int `yaml:"max_upload_parts,omitempty"` + MinChunkSize *int64 `yaml:"min_chunk_size,omitempty"` } // Provider defines the configurable data in each provider.yaml diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 137b31adb..c12a97619 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -39,6 +39,9 @@ import ( smithyhttp "github.com/aws/smithy-go/transport/http" "github.com/ncw/swift/v2" + "golang.org/x/net/http/httpguts" + "golang.org/x/sync/errgroup" + "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/chunksize" @@ -59,8 +62,6 @@ import ( "github.com/rclone/rclone/lib/readers" "github.com/rclone/rclone/lib/rest" "github.com/rclone/rclone/lib/version" - "golang.org/x/net/http/httpguts" - "golang.org/x/sync/errgroup" ) // Register with Fs @@ -574,6 +575,13 @@ circumstances or for testing. `, Default: false, Advanced: true, + }, { + Name: "use_data_integrity_protections", + Help: `If true use AWS S3 data integrity protections. + +See [AWS Docs on Data Integrity Protections](https://docs.aws.amazon.com/sdkref/latest/guide/feature-dataintegrity.html)`, + Default: fs.Tristate{}, + Advanced: true, }, { Name: "versions", Help: "Include old versions in directory listings.", @@ -892,67 +900,68 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{ // Options defines the configuration for this backend type Options struct { - Provider string `config:"provider"` - EnvAuth bool `config:"env_auth"` - AccessKeyID string `config:"access_key_id"` - SecretAccessKey string `config:"secret_access_key"` - Region string `config:"region"` - Endpoint string `config:"endpoint"` - STSEndpoint string `config:"sts_endpoint"` - UseDualStack bool `config:"use_dual_stack"` - LocationConstraint string `config:"location_constraint"` - ACL string `config:"acl"` - BucketACL string `config:"bucket_acl"` - RequesterPays bool `config:"requester_pays"` - ServerSideEncryption string `config:"server_side_encryption"` - SSEKMSKeyID string `config:"sse_kms_key_id"` - SSECustomerAlgorithm string `config:"sse_customer_algorithm"` - SSECustomerKey string `config:"sse_customer_key"` - SSECustomerKeyBase64 string `config:"sse_customer_key_base64"` - SSECustomerKeyMD5 string `config:"sse_customer_key_md5"` - StorageClass string `config:"storage_class"` - UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` - CopyCutoff fs.SizeSuffix `config:"copy_cutoff"` - ChunkSize fs.SizeSuffix `config:"chunk_size"` - MaxUploadParts int `config:"max_upload_parts"` - DisableChecksum bool `config:"disable_checksum"` - SharedCredentialsFile string `config:"shared_credentials_file"` - Profile string `config:"profile"` - SessionToken string `config:"session_token"` - UploadConcurrency int `config:"upload_concurrency"` - ForcePathStyle bool `config:"force_path_style"` - V2Auth bool `config:"v2_auth"` - UseAccelerateEndpoint bool `config:"use_accelerate_endpoint"` - UseARNRegion bool `config:"use_arn_region"` - LeavePartsOnError bool `config:"leave_parts_on_error"` - ListChunk int32 `config:"list_chunk"` - ListVersion int `config:"list_version"` - ListURLEncode fs.Tristate `config:"list_url_encode"` - NoCheckBucket bool `config:"no_check_bucket"` - NoHead bool `config:"no_head"` - NoHeadObject bool `config:"no_head_object"` - Enc encoder.MultiEncoder `config:"encoding"` - DisableHTTP2 bool `config:"disable_http2"` - DownloadURL string `config:"download_url"` - DirectoryMarkers bool `config:"directory_markers"` - UseMultipartEtag fs.Tristate `config:"use_multipart_etag"` - UsePresignedRequest bool `config:"use_presigned_request"` - Versions bool `config:"versions"` - VersionAt fs.Time `config:"version_at"` - VersionDeleted bool `config:"version_deleted"` - Decompress bool `config:"decompress"` - MightGzip fs.Tristate `config:"might_gzip"` - UseAcceptEncodingGzip fs.Tristate `config:"use_accept_encoding_gzip"` - NoSystemMetadata bool `config:"no_system_metadata"` - UseAlreadyExists fs.Tristate `config:"use_already_exists"` - UseMultipartUploads fs.Tristate `config:"use_multipart_uploads"` - UseUnsignedPayload fs.Tristate `config:"use_unsigned_payload"` - SDKLogMode sdkLogMode `config:"sdk_log_mode"` - DirectoryBucket bool `config:"directory_bucket"` - IBMAPIKey string `config:"ibm_api_key"` - IBMInstanceID string `config:"ibm_resource_instance_id"` - UseXID fs.Tristate `config:"use_x_id"` - SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"` + Provider string `config:"provider"` + EnvAuth bool `config:"env_auth"` + AccessKeyID string `config:"access_key_id"` + SecretAccessKey string `config:"secret_access_key"` + Region string `config:"region"` + Endpoint string `config:"endpoint"` + STSEndpoint string `config:"sts_endpoint"` + UseDualStack bool `config:"use_dual_stack"` + LocationConstraint string `config:"location_constraint"` + ACL string `config:"acl"` + BucketACL string `config:"bucket_acl"` + RequesterPays bool `config:"requester_pays"` + ServerSideEncryption string `config:"server_side_encryption"` + SSEKMSKeyID string `config:"sse_kms_key_id"` + SSECustomerAlgorithm string `config:"sse_customer_algorithm"` + SSECustomerKey string `config:"sse_customer_key"` + SSECustomerKeyBase64 string `config:"sse_customer_key_base64"` + SSECustomerKeyMD5 string `config:"sse_customer_key_md5"` + StorageClass string `config:"storage_class"` + UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` + CopyCutoff fs.SizeSuffix `config:"copy_cutoff"` + ChunkSize fs.SizeSuffix `config:"chunk_size"` + MaxUploadParts int `config:"max_upload_parts"` + DisableChecksum bool `config:"disable_checksum"` + SharedCredentialsFile string `config:"shared_credentials_file"` + Profile string `config:"profile"` + SessionToken string `config:"session_token"` + UploadConcurrency int `config:"upload_concurrency"` + ForcePathStyle bool `config:"force_path_style"` + V2Auth bool `config:"v2_auth"` + UseAccelerateEndpoint bool `config:"use_accelerate_endpoint"` + UseARNRegion bool `config:"use_arn_region"` + LeavePartsOnError bool `config:"leave_parts_on_error"` + ListChunk int32 `config:"list_chunk"` + ListVersion int `config:"list_version"` + ListURLEncode fs.Tristate `config:"list_url_encode"` + NoCheckBucket bool `config:"no_check_bucket"` + NoHead bool `config:"no_head"` + NoHeadObject bool `config:"no_head_object"` + Enc encoder.MultiEncoder `config:"encoding"` + DisableHTTP2 bool `config:"disable_http2"` + DownloadURL string `config:"download_url"` + DirectoryMarkers bool `config:"directory_markers"` + UseMultipartEtag fs.Tristate `config:"use_multipart_etag"` + UsePresignedRequest bool `config:"use_presigned_request"` + UseDataIntegrityProtections fs.Tristate `config:"use_data_integrity_protections"` + Versions bool `config:"versions"` + VersionAt fs.Time `config:"version_at"` + VersionDeleted bool `config:"version_deleted"` + Decompress bool `config:"decompress"` + MightGzip fs.Tristate `config:"might_gzip"` + UseAcceptEncodingGzip fs.Tristate `config:"use_accept_encoding_gzip"` + NoSystemMetadata bool `config:"no_system_metadata"` + UseAlreadyExists fs.Tristate `config:"use_already_exists"` + UseMultipartUploads fs.Tristate `config:"use_multipart_uploads"` + UseUnsignedPayload fs.Tristate `config:"use_unsigned_payload"` + SDKLogMode sdkLogMode `config:"sdk_log_mode"` + DirectoryBucket bool `config:"directory_bucket"` + IBMAPIKey string `config:"ibm_api_key"` + IBMInstanceID string `config:"ibm_resource_instance_id"` + UseXID fs.Tristate `config:"use_x_id"` + SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"` } // Fs represents a remote s3 server @@ -1302,6 +1311,10 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli } else { s3Opt.EndpointOptions.UseDualStackEndpoint = aws.DualStackEndpointStateDisabled } + if !opt.UseDataIntegrityProtections.Value { + s3Opt.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + s3Opt.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired + } // FIXME not ported from SDK v1 - not sure what this does // s3Opt.UsEast1RegionalEndpoint = endpoints.RegionalS3UsEast1Endpoint }) @@ -1497,6 +1510,7 @@ func setQuirks(opt *Options, provider *Provider) { set(&opt.ListURLEncode, true, provider.Quirks.ListURLEncode) set(&opt.UseMultipartEtag, true, provider.Quirks.UseMultipartEtag) set(&opt.UseAcceptEncodingGzip, true, provider.Quirks.UseAcceptEncodingGzip) + set(&opt.UseDataIntegrityProtections, false, provider.Quirks.UseDataIntegrityProtections) set(&opt.MightGzip, true, provider.Quirks.MightGzip) set(&opt.UseAlreadyExists, true, provider.Quirks.UseAlreadyExists) set(&opt.UseMultipartUploads, true, provider.Quirks.UseMultipartUploads)