From cb1507fa96a39d03f80e1da934e72464e49edf61 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Sat, 16 Aug 2025 10:36:50 +0100
Subject: [PATCH] s3: fix Content-Type: aws-chunked causing upload errors with
 --metadata

`Content-Type: aws-chunked` is used on S3 PUT requests to signal SigV4
streaming uploads: the body is sent in AWS-formatted chunks, each
chunk framed and HMAC-signed.

When copying from a non S3 compatible object store (like Digital
Ocean) the objects can have `Content-Type: aws-chunked` (which you
won't see on AWS S3). Attempting to copy these objects to S3 with
`--metadata` this produces this error.

    aws-chunked encoding is not supported when x-amz-content-sha256 UNSIGNED-PAYLOAD is supplied

This patch makes sure `aws-chunked` is removed from the `Content-Type`
metadata both on the way in and the way out.

Fixes #8724
---
 backend/s3/s3.go               | 38 +++++++++++++++++++++++++++----
 backend/s3/s3_internal_test.go | 41 ++++++++++++++++++++++++++++++++++
 2 files changed, 75 insertions(+), 4 deletions(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index a1379da3d..de06383af 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -6086,7 +6086,7 @@ func (o *Object) setMetaData(resp *s3.HeadObjectOutput) {
 	o.storageClass = stringClone(string(resp.StorageClass))
 	o.cacheControl = stringClonePointer(resp.CacheControl)
 	o.contentDisposition = stringClonePointer(resp.ContentDisposition)
-	o.contentEncoding = stringClonePointer(resp.ContentEncoding)
+	o.contentEncoding = stringClonePointer(removeAWSChunked(resp.ContentEncoding))
 	o.contentLanguage = stringClonePointer(resp.ContentLanguage)
 
 	// If decompressing then size and md5sum are unknown
@@ -6154,6 +6154,36 @@ func (o *Object) Storable() bool {
 	return true
 }
 
+// removeAWSChunked removes the "aws-chunked" content-coding from a
+// Content-Encoding field value (RFC 9110). Comparison is case-insensitive.
+// Returns nil if encoding is empty after removal.
+func removeAWSChunked(pv *string) *string {
+	if pv == nil {
+		return nil
+	}
+	v := *pv
+	if v == "" {
+		return nil
+	}
+	if !strings.Contains(strings.ToLower(v), "aws-chunked") {
+		return pv
+	}
+	parts := strings.Split(v, ",")
+	out := make([]string, 0, len(parts))
+	for _, p := range parts {
+		tok := strings.TrimSpace(p)
+		if tok == "" || strings.EqualFold(tok, "aws-chunked") {
+			continue
+		}
+		out = append(out, tok)
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	v = strings.Join(out, ",")
+	return &v
+}
+
 func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	url := o.fs.opt.DownloadURL + bucketPath
 	var resp *http.Response
@@ -6322,7 +6352,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	o.setMetaData(&head)
 
 	// Decompress body if necessary
-	if deref(resp.ContentEncoding) == "gzip" {
+	if deref(removeAWSChunked(resp.ContentEncoding)) == "gzip" {
 		if o.fs.opt.Decompress || (resp.ContentLength == nil && o.fs.opt.MightGzip.Value) {
 			return readers.NewGzipReader(resp.Body)
 		}
@@ -6746,7 +6776,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
 		case "content-disposition":
 			ui.req.ContentDisposition = pv
 		case "content-encoding":
-			ui.req.ContentEncoding = pv
+			ui.req.ContentEncoding = removeAWSChunked(pv)
 		case "content-language":
 			ui.req.ContentLanguage = pv
 		case "content-type":
@@ -6843,7 +6873,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
 		case "content-disposition":
 			ui.req.ContentDisposition = aws.String(value)
 		case "content-encoding":
-			ui.req.ContentEncoding = aws.String(value)
+			ui.req.ContentEncoding = removeAWSChunked(aws.String(value))
 		case "content-language":
 			ui.req.ContentLanguage = aws.String(value)
 		case "content-type":
diff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.go
index 86e23e436..cdca649ae 100644
--- a/backend/s3/s3_internal_test.go
+++ b/backend/s3/s3_internal_test.go
@@ -248,6 +248,47 @@ func TestMergeDeleteMarkers(t *testing.T) {
 	}
 }
 
+func TestRemoveAWSChunked(t *testing.T) {
+	ps := func(s string) *string {
+		return &s
+	}
+	tests := []struct {
+		name string
+		in   *string
+		want *string
+	}{
+		{"nil", nil, nil},
+		{"empty", ps(""), nil},
+		{"only aws", ps("aws-chunked"), nil},
+		{"leading aws", ps("aws-chunked, gzip"), ps("gzip")},
+		{"trailing aws", ps("gzip, aws-chunked"), ps("gzip")},
+		{"middle aws", ps("gzip, aws-chunked, br"), ps("gzip,br")},
+		{"case insensitive", ps("GZip, AwS-ChUnKeD, Br"), ps("GZip,Br")},
+		{"duplicates", ps("aws-chunked , aws-chunked"), nil},
+		{"no aws normalize spaces", ps(" gzip ,  br "), ps(" gzip ,  br ")},
+		{"surrounding spaces", ps("  aws-chunked  "), nil},
+		{"no change", ps("gzip, br"), ps("gzip, br")},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := removeAWSChunked(tc.in)
+			check := func(want, got *string) {
+				t.Helper()
+				if tc.want == nil {
+					assert.Nil(t, got)
+				} else {
+					require.NotNil(t, got)
+					assert.Equal(t, *tc.want, *got)
+				}
+			}
+			check(tc.want, got)
+			// Idempotent
+			got2 := removeAWSChunked(got)
+			check(got, got2)
+		})
+	}
+}
+
 func (f *Fs) InternalTestVersions(t *testing.T) {
 	ctx := context.Background()