mirror of
https://github.com/rclone/rclone.git
synced 2025-12-11 22:14:05 +01:00
s3: fix Content-Type: aws-chunked causing upload errors with --metadata
`Content-Type: aws-chunked` is used on S3 PUT requests to signal SigV4
streaming uploads: the body is sent in AWS-formatted chunks, each
chunk framed and HMAC-signed.
When copying from a non S3 compatible object store (like Digital
Ocean) the objects can have `Content-Type: aws-chunked` (which you
won't see on AWS S3). Attempting to copy these objects to S3 with
`--metadata` this produces this error.
aws-chunked encoding is not supported when x-amz-content-sha256 UNSIGNED-PAYLOAD is supplied
This patch makes sure `aws-chunked` is removed from the `Content-Type`
metadata both on the way in and the way out.
Fixes #8724
This commit is contained in:
@@ -248,6 +248,47 @@ func TestMergeDeleteMarkers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveAWSChunked(t *testing.T) {
|
||||
ps := func(s string) *string {
|
||||
return &s
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
in *string
|
||||
want *string
|
||||
}{
|
||||
{"nil", nil, nil},
|
||||
{"empty", ps(""), nil},
|
||||
{"only aws", ps("aws-chunked"), nil},
|
||||
{"leading aws", ps("aws-chunked, gzip"), ps("gzip")},
|
||||
{"trailing aws", ps("gzip, aws-chunked"), ps("gzip")},
|
||||
{"middle aws", ps("gzip, aws-chunked, br"), ps("gzip,br")},
|
||||
{"case insensitive", ps("GZip, AwS-ChUnKeD, Br"), ps("GZip,Br")},
|
||||
{"duplicates", ps("aws-chunked , aws-chunked"), nil},
|
||||
{"no aws normalize spaces", ps(" gzip , br "), ps(" gzip , br ")},
|
||||
{"surrounding spaces", ps(" aws-chunked "), nil},
|
||||
{"no change", ps("gzip, br"), ps("gzip, br")},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := removeAWSChunked(tc.in)
|
||||
check := func(want, got *string) {
|
||||
t.Helper()
|
||||
if tc.want == nil {
|
||||
assert.Nil(t, got)
|
||||
} else {
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, *tc.want, *got)
|
||||
}
|
||||
}
|
||||
check(tc.want, got)
|
||||
// Idempotent
|
||||
got2 := removeAWSChunked(got)
|
||||
check(got, got2)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTestVersions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user