mirror of
https://github.com/rclone/rclone.git
synced 2025-12-11 22:14:05 +01:00
s3: fix Content-Type: aws-chunked causing upload errors with --metadata
`Content-Type: aws-chunked` is used on S3 PUT requests to signal SigV4
streaming uploads: the body is sent in AWS-formatted chunks, each
chunk framed and HMAC-signed.
When copying from a non S3 compatible object store (like Digital
Ocean) the objects can have `Content-Type: aws-chunked` (which you
won't see on AWS S3). Attempting to copy these objects to S3 with
`--metadata` this produces this error.
aws-chunked encoding is not supported when x-amz-content-sha256 UNSIGNED-PAYLOAD is supplied
This patch makes sure `aws-chunked` is removed from the `Content-Type`
metadata both on the way in and the way out.
Fixes #8724
This commit is contained in:
@@ -6086,7 +6086,7 @@ func (o *Object) setMetaData(resp *s3.HeadObjectOutput) {
|
|||||||
o.storageClass = stringClone(string(resp.StorageClass))
|
o.storageClass = stringClone(string(resp.StorageClass))
|
||||||
o.cacheControl = stringClonePointer(resp.CacheControl)
|
o.cacheControl = stringClonePointer(resp.CacheControl)
|
||||||
o.contentDisposition = stringClonePointer(resp.ContentDisposition)
|
o.contentDisposition = stringClonePointer(resp.ContentDisposition)
|
||||||
o.contentEncoding = stringClonePointer(resp.ContentEncoding)
|
o.contentEncoding = stringClonePointer(removeAWSChunked(resp.ContentEncoding))
|
||||||
o.contentLanguage = stringClonePointer(resp.ContentLanguage)
|
o.contentLanguage = stringClonePointer(resp.ContentLanguage)
|
||||||
|
|
||||||
// If decompressing then size and md5sum are unknown
|
// If decompressing then size and md5sum are unknown
|
||||||
@@ -6154,6 +6154,36 @@ func (o *Object) Storable() bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// removeAWSChunked removes the "aws-chunked" content-coding from a
|
||||||
|
// Content-Encoding field value (RFC 9110). Comparison is case-insensitive.
|
||||||
|
// Returns nil if encoding is empty after removal.
|
||||||
|
func removeAWSChunked(pv *string) *string {
|
||||||
|
if pv == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
v := *pv
|
||||||
|
if v == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !strings.Contains(strings.ToLower(v), "aws-chunked") {
|
||||||
|
return pv
|
||||||
|
}
|
||||||
|
parts := strings.Split(v, ",")
|
||||||
|
out := make([]string, 0, len(parts))
|
||||||
|
for _, p := range parts {
|
||||||
|
tok := strings.TrimSpace(p)
|
||||||
|
if tok == "" || strings.EqualFold(tok, "aws-chunked") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, tok)
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
v = strings.Join(out, ",")
|
||||||
|
return &v
|
||||||
|
}
|
||||||
|
|
||||||
func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||||
url := o.fs.opt.DownloadURL + bucketPath
|
url := o.fs.opt.DownloadURL + bucketPath
|
||||||
var resp *http.Response
|
var resp *http.Response
|
||||||
@@ -6322,7 +6352,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
|||||||
o.setMetaData(&head)
|
o.setMetaData(&head)
|
||||||
|
|
||||||
// Decompress body if necessary
|
// Decompress body if necessary
|
||||||
if deref(resp.ContentEncoding) == "gzip" {
|
if deref(removeAWSChunked(resp.ContentEncoding)) == "gzip" {
|
||||||
if o.fs.opt.Decompress || (resp.ContentLength == nil && o.fs.opt.MightGzip.Value) {
|
if o.fs.opt.Decompress || (resp.ContentLength == nil && o.fs.opt.MightGzip.Value) {
|
||||||
return readers.NewGzipReader(resp.Body)
|
return readers.NewGzipReader(resp.Body)
|
||||||
}
|
}
|
||||||
@@ -6746,7 +6776,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
|||||||
case "content-disposition":
|
case "content-disposition":
|
||||||
ui.req.ContentDisposition = pv
|
ui.req.ContentDisposition = pv
|
||||||
case "content-encoding":
|
case "content-encoding":
|
||||||
ui.req.ContentEncoding = pv
|
ui.req.ContentEncoding = removeAWSChunked(pv)
|
||||||
case "content-language":
|
case "content-language":
|
||||||
ui.req.ContentLanguage = pv
|
ui.req.ContentLanguage = pv
|
||||||
case "content-type":
|
case "content-type":
|
||||||
@@ -6843,7 +6873,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
|||||||
case "content-disposition":
|
case "content-disposition":
|
||||||
ui.req.ContentDisposition = aws.String(value)
|
ui.req.ContentDisposition = aws.String(value)
|
||||||
case "content-encoding":
|
case "content-encoding":
|
||||||
ui.req.ContentEncoding = aws.String(value)
|
ui.req.ContentEncoding = removeAWSChunked(aws.String(value))
|
||||||
case "content-language":
|
case "content-language":
|
||||||
ui.req.ContentLanguage = aws.String(value)
|
ui.req.ContentLanguage = aws.String(value)
|
||||||
case "content-type":
|
case "content-type":
|
||||||
|
|||||||
@@ -248,6 +248,47 @@ func TestMergeDeleteMarkers(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRemoveAWSChunked(t *testing.T) {
|
||||||
|
ps := func(s string) *string {
|
||||||
|
return &s
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
in *string
|
||||||
|
want *string
|
||||||
|
}{
|
||||||
|
{"nil", nil, nil},
|
||||||
|
{"empty", ps(""), nil},
|
||||||
|
{"only aws", ps("aws-chunked"), nil},
|
||||||
|
{"leading aws", ps("aws-chunked, gzip"), ps("gzip")},
|
||||||
|
{"trailing aws", ps("gzip, aws-chunked"), ps("gzip")},
|
||||||
|
{"middle aws", ps("gzip, aws-chunked, br"), ps("gzip,br")},
|
||||||
|
{"case insensitive", ps("GZip, AwS-ChUnKeD, Br"), ps("GZip,Br")},
|
||||||
|
{"duplicates", ps("aws-chunked , aws-chunked"), nil},
|
||||||
|
{"no aws normalize spaces", ps(" gzip , br "), ps(" gzip , br ")},
|
||||||
|
{"surrounding spaces", ps(" aws-chunked "), nil},
|
||||||
|
{"no change", ps("gzip, br"), ps("gzip, br")},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got := removeAWSChunked(tc.in)
|
||||||
|
check := func(want, got *string) {
|
||||||
|
t.Helper()
|
||||||
|
if tc.want == nil {
|
||||||
|
assert.Nil(t, got)
|
||||||
|
} else {
|
||||||
|
require.NotNil(t, got)
|
||||||
|
assert.Equal(t, *tc.want, *got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
check(tc.want, got)
|
||||||
|
// Idempotent
|
||||||
|
got2 := removeAWSChunked(got)
|
||||||
|
check(got, got2)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (f *Fs) InternalTestVersions(t *testing.T) {
|
func (f *Fs) InternalTestVersions(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user