s3: update to using AWS SDK v2 - fixes #4989

SDK v2 conversion

Changes

  - `--s3-sts-endpoint` is no longer supported
  - `--s3-use-unsigned-payload` to control use of trailer checksums (needed for non AWS)
This commit is contained in:
Nick Craig-Wood
2024-08-03 11:35:32 +01:00
parent a1f52bcf50
commit fd9049c83d
9 changed files with 558 additions and 552 deletions

View File

@@ -13,7 +13,8 @@ import (
"reflect"
"strings"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
)
// flags
@@ -82,15 +83,18 @@ func main() {
package s3
import "github.com/aws/aws-sdk-go/service/s3"
import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
)
`)
genSetFrom(new(s3.ListObjectsInput), new(s3.ListObjectsV2Input))
genSetFrom(new(s3.ListObjectsV2Output), new(s3.ListObjectsOutput))
genSetFrom(new(s3.ListObjectVersionsInput), new(s3.ListObjectsV2Input))
genSetFrom(new(s3.ObjectVersion), new(s3.DeleteMarkerEntry))
genSetFrom(new(types.ObjectVersion), new(types.DeleteMarkerEntry))
genSetFrom(new(s3.ListObjectsV2Output), new(s3.ListObjectVersionsOutput))
genSetFrom(new(s3.Object), new(s3.ObjectVersion))
genSetFrom(new(types.Object), new(types.ObjectVersion))
genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.HeadObjectOutput))
genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.CopyObjectInput))
genSetFrom(new(s3.UploadPartCopyInput), new(s3.CopyObjectInput))

File diff suppressed because it is too large Load Diff

View File

@@ -5,15 +5,17 @@ import (
"compress/gzip"
"context"
"crypto/md5"
"errors"
"fmt"
"path"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fs/hash"
@@ -131,20 +133,20 @@ func TestVersionLess(t *testing.T) {
t1 := fstest.Time("2022-01-21T12:00:00+01:00")
t2 := fstest.Time("2022-01-21T12:00:01+01:00")
for n, test := range []struct {
a, b *s3.ObjectVersion
a, b *types.ObjectVersion
want bool
}{
{a: nil, b: nil, want: true},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: nil, want: false},
{a: nil, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: true},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: false},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t2}, want: false},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t2}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: true},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key2, LastModified: &t1}, want: true},
{a: &s3.ObjectVersion{Key: &key2, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: false},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(false)}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: false},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(true)}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: true},
{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(false)}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(true)}, want: false},
{a: &types.ObjectVersion{Key: &key1, LastModified: &t1}, b: nil, want: false},
{a: nil, b: &types.ObjectVersion{Key: &key1, LastModified: &t1}, want: true},
{a: &types.ObjectVersion{Key: &key1, LastModified: &t1}, b: &types.ObjectVersion{Key: &key1, LastModified: &t1}, want: false},
{a: &types.ObjectVersion{Key: &key1, LastModified: &t1}, b: &types.ObjectVersion{Key: &key1, LastModified: &t2}, want: false},
{a: &types.ObjectVersion{Key: &key1, LastModified: &t2}, b: &types.ObjectVersion{Key: &key1, LastModified: &t1}, want: true},
{a: &types.ObjectVersion{Key: &key1, LastModified: &t1}, b: &types.ObjectVersion{Key: &key2, LastModified: &t1}, want: true},
{a: &types.ObjectVersion{Key: &key2, LastModified: &t1}, b: &types.ObjectVersion{Key: &key1, LastModified: &t1}, want: false},
{a: &types.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(false)}, b: &types.ObjectVersion{Key: &key1, LastModified: &t1}, want: false},
{a: &types.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(true)}, b: &types.ObjectVersion{Key: &key1, LastModified: &t1}, want: true},
{a: &types.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(false)}, b: &types.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(true)}, want: false},
} {
got := versionLess(test.a, test.b)
assert.Equal(t, test.want, got, fmt.Sprintf("%d: %+v", n, test))
@@ -157,24 +159,24 @@ func TestMergeDeleteMarkers(t *testing.T) {
t1 := fstest.Time("2022-01-21T12:00:00+01:00")
t2 := fstest.Time("2022-01-21T12:00:01+01:00")
for n, test := range []struct {
versions []*s3.ObjectVersion
markers []*s3.DeleteMarkerEntry
want []*s3.ObjectVersion
versions []types.ObjectVersion
markers []types.DeleteMarkerEntry
want []types.ObjectVersion
}{
{
versions: []*s3.ObjectVersion{},
markers: []*s3.DeleteMarkerEntry{},
want: []*s3.ObjectVersion{},
versions: []types.ObjectVersion{},
markers: []types.DeleteMarkerEntry{},
want: []types.ObjectVersion{},
},
{
versions: []*s3.ObjectVersion{
versions: []types.ObjectVersion{
{
Key: &key1,
LastModified: &t1,
},
},
markers: []*s3.DeleteMarkerEntry{},
want: []*s3.ObjectVersion{
markers: []types.DeleteMarkerEntry{},
want: []types.ObjectVersion{
{
Key: &key1,
LastModified: &t1,
@@ -182,14 +184,14 @@ func TestMergeDeleteMarkers(t *testing.T) {
},
},
{
versions: []*s3.ObjectVersion{},
markers: []*s3.DeleteMarkerEntry{
versions: []types.ObjectVersion{},
markers: []types.DeleteMarkerEntry{
{
Key: &key1,
LastModified: &t1,
},
},
want: []*s3.ObjectVersion{
want: []types.ObjectVersion{
{
Key: &key1,
LastModified: &t1,
@@ -198,7 +200,7 @@ func TestMergeDeleteMarkers(t *testing.T) {
},
},
{
versions: []*s3.ObjectVersion{
versions: []types.ObjectVersion{
{
Key: &key1,
LastModified: &t2,
@@ -208,13 +210,13 @@ func TestMergeDeleteMarkers(t *testing.T) {
LastModified: &t2,
},
},
markers: []*s3.DeleteMarkerEntry{
markers: []types.DeleteMarkerEntry{
{
Key: &key1,
LastModified: &t1,
},
},
want: []*s3.ObjectVersion{
want: []types.ObjectVersion{
{
Key: &key1,
LastModified: &t2,
@@ -399,22 +401,23 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
// quirk is set correctly
req := s3.CreateBucketInput{
Bucket: &f.rootBucket,
ACL: stringPointerOrNil(f.opt.BucketACL),
ACL: types.BucketCannedACL(f.opt.BucketACL),
}
if f.opt.LocationConstraint != "" {
req.CreateBucketConfiguration = &s3.CreateBucketConfiguration{
LocationConstraint: &f.opt.LocationConstraint,
req.CreateBucketConfiguration = &types.CreateBucketConfiguration{
LocationConstraint: types.BucketLocationConstraint(f.opt.LocationConstraint),
}
}
err := f.pacer.Call(func() (bool, error) {
_, err := f.c.CreateBucketWithContext(ctx, &req)
_, err := f.c.CreateBucket(ctx, &req)
return f.shouldRetry(ctx, err)
})
var errString string
var awsError smithy.APIError
if err == nil {
errString = "No Error"
} else if awsErr, ok := err.(awserr.Error); ok {
errString = awsErr.Code()
} else if errors.As(err, &awsError) {
errString = awsError.ErrorCode()
} else {
assert.Fail(t, "Unknown error %T %v", err, err)
}

View File

@@ -4,12 +4,14 @@ package s3
import (
"context"
"net/http"
"strings"
"testing"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func SetupS3Test(t *testing.T) (context.Context, *Options, *http.Client) {
@@ -54,20 +56,16 @@ func TestAWSDualStackOption(t *testing.T) {
// test enabled
ctx, opt, client := SetupS3Test(t)
opt.UseDualStack = true
s3Conn, _, _ := s3Connection(ctx, opt, client)
if !strings.Contains(s3Conn.Endpoint, "dualstack") {
t.Errorf("dualstack failed got: %s, wanted: dualstack", s3Conn.Endpoint)
t.Fail()
}
s3Conn, err := s3Connection(ctx, opt, client)
require.NoError(t, err)
assert.Equal(t, aws.DualStackEndpointStateEnabled, s3Conn.Options().EndpointOptions.UseDualStackEndpoint)
}
{
// test default case
ctx, opt, client := SetupS3Test(t)
s3Conn, _, _ := s3Connection(ctx, opt, client)
if strings.Contains(s3Conn.Endpoint, "dualstack") {
t.Errorf("dualstack failed got: %s, NOT wanted: dualstack", s3Conn.Endpoint)
t.Fail()
}
s3Conn, err := s3Connection(ctx, opt, client)
require.NoError(t, err)
assert.Equal(t, aws.DualStackEndpointStateDisabled, s3Conn.Options().EndpointOptions.UseDualStackEndpoint)
}
}

View File

@@ -2,7 +2,10 @@
package s3
import "github.com/aws/aws-sdk-go/service/s3"
import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
)
// setFrom_s3ListObjectsInput_s3ListObjectsV2Input copies matching elements from a to b
func setFrom_s3ListObjectsInput_s3ListObjectsV2Input(a *s3.ListObjectsInput, b *s3.ListObjectsV2Input) {
@@ -27,6 +30,7 @@ func setFrom_s3ListObjectsV2Output_s3ListObjectsOutput(a *s3.ListObjectsV2Output
a.Name = b.Name
a.Prefix = b.Prefix
a.RequestCharged = b.RequestCharged
a.ResultMetadata = b.ResultMetadata
}
// setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input copies matching elements from a to b
@@ -41,8 +45,8 @@ func setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input(a *s3.ListObjectVers
a.RequestPayer = b.RequestPayer
}
// setFrom_s3ObjectVersion_s3DeleteMarkerEntry copies matching elements from a to b
func setFrom_s3ObjectVersion_s3DeleteMarkerEntry(a *s3.ObjectVersion, b *s3.DeleteMarkerEntry) {
// setFrom_typesObjectVersion_typesDeleteMarkerEntry copies matching elements from a to b
func setFrom_typesObjectVersion_typesDeleteMarkerEntry(a *types.ObjectVersion, b *types.DeleteMarkerEntry) {
a.IsLatest = b.IsLatest
a.Key = b.Key
a.LastModified = b.LastModified
@@ -60,10 +64,11 @@ func setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(a *s3.ListObjectsV
a.Name = b.Name
a.Prefix = b.Prefix
a.RequestCharged = b.RequestCharged
a.ResultMetadata = b.ResultMetadata
}
// setFrom_s3Object_s3ObjectVersion copies matching elements from a to b
func setFrom_s3Object_s3ObjectVersion(a *s3.Object, b *s3.ObjectVersion) {
// setFrom_typesObject_typesObjectVersion copies matching elements from a to b
func setFrom_typesObject_typesObjectVersion(a *types.Object, b *types.ObjectVersion) {
a.ChecksumAlgorithm = b.ChecksumAlgorithm
a.ETag = b.ETag
a.Key = b.Key
@@ -71,7 +76,6 @@ func setFrom_s3Object_s3ObjectVersion(a *s3.Object, b *s3.ObjectVersion) {
a.Owner = b.Owner
a.RestoreStatus = b.RestoreStatus
a.Size = b.Size
a.StorageClass = b.StorageClass
}
// setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput copies matching elements from a to b
@@ -82,6 +86,7 @@ func setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput(a *s3.CreateMultipa
a.ContentEncoding = b.ContentEncoding
a.ContentLanguage = b.ContentLanguage
a.ContentType = b.ContentType
a.Expires = b.Expires
a.Metadata = b.Metadata
a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
a.ObjectLockMode = b.ObjectLockMode
@@ -96,8 +101,9 @@ func setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput(a *s3.CreateMultipa
// setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput copies matching elements from a to b
func setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput(a *s3.CreateMultipartUploadInput, b *s3.CopyObjectInput) {
a.ACL = b.ACL
a.Bucket = b.Bucket
a.Key = b.Key
a.ACL = b.ACL
a.BucketKeyEnabled = b.BucketKeyEnabled
a.CacheControl = b.CacheControl
a.ChecksumAlgorithm = b.ChecksumAlgorithm
@@ -111,7 +117,6 @@ func setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput(a *s3.CreateMultipar
a.GrantRead = b.GrantRead
a.GrantReadACP = b.GrantReadACP
a.GrantWriteACP = b.GrantWriteACP
a.Key = b.Key
a.Metadata = b.Metadata
a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
a.ObjectLockMode = b.ObjectLockMode
@@ -132,6 +137,7 @@ func setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput(a *s3.CreateMultipar
func setFrom_s3UploadPartCopyInput_s3CopyObjectInput(a *s3.UploadPartCopyInput, b *s3.CopyObjectInput) {
a.Bucket = b.Bucket
a.CopySource = b.CopySource
a.Key = b.Key
a.CopySourceIfMatch = b.CopySourceIfMatch
a.CopySourceIfModifiedSince = b.CopySourceIfModifiedSince
a.CopySourceIfNoneMatch = b.CopySourceIfNoneMatch
@@ -141,7 +147,6 @@ func setFrom_s3UploadPartCopyInput_s3CopyObjectInput(a *s3.UploadPartCopyInput,
a.CopySourceSSECustomerKeyMD5 = b.CopySourceSSECustomerKeyMD5
a.ExpectedBucketOwner = b.ExpectedBucketOwner
a.ExpectedSourceBucketOwner = b.ExpectedSourceBucketOwner
a.Key = b.Key
a.RequestPayer = b.RequestPayer
a.SSECustomerAlgorithm = b.SSECustomerAlgorithm
a.SSECustomerKey = b.SSECustomerKey
@@ -166,6 +171,7 @@ func setFrom_s3HeadObjectOutput_s3GetObjectOutput(a *s3.HeadObjectOutput, b *s3.
a.ETag = b.ETag
a.Expiration = b.Expiration
a.Expires = b.Expires
a.ExpiresString = b.ExpiresString
a.LastModified = b.LastModified
a.Metadata = b.Metadata
a.MissingMeta = b.MissingMeta
@@ -183,12 +189,14 @@ func setFrom_s3HeadObjectOutput_s3GetObjectOutput(a *s3.HeadObjectOutput, b *s3.
a.StorageClass = b.StorageClass
a.VersionId = b.VersionId
a.WebsiteRedirectLocation = b.WebsiteRedirectLocation
a.ResultMetadata = b.ResultMetadata
}
// setFrom_s3CreateMultipartUploadInput_s3PutObjectInput copies matching elements from a to b
func setFrom_s3CreateMultipartUploadInput_s3PutObjectInput(a *s3.CreateMultipartUploadInput, b *s3.PutObjectInput) {
a.ACL = b.ACL
a.Bucket = b.Bucket
a.Key = b.Key
a.ACL = b.ACL
a.BucketKeyEnabled = b.BucketKeyEnabled
a.CacheControl = b.CacheControl
a.ChecksumAlgorithm = b.ChecksumAlgorithm
@@ -202,7 +210,6 @@ func setFrom_s3CreateMultipartUploadInput_s3PutObjectInput(a *s3.CreateMultipart
a.GrantRead = b.GrantRead
a.GrantReadACP = b.GrantReadACP
a.GrantWriteACP = b.GrantWriteACP
a.Key = b.Key
a.Metadata = b.Metadata
a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
a.ObjectLockMode = b.ObjectLockMode
@@ -232,6 +239,7 @@ func setFrom_s3HeadObjectOutput_s3PutObjectInput(a *s3.HeadObjectOutput, b *s3.P
a.ContentLanguage = b.ContentLanguage
a.ContentLength = b.ContentLength
a.ContentType = b.ContentType
a.Expires = b.Expires
a.Metadata = b.Metadata
a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
a.ObjectLockMode = b.ObjectLockMode
@@ -246,8 +254,9 @@ func setFrom_s3HeadObjectOutput_s3PutObjectInput(a *s3.HeadObjectOutput, b *s3.P
// setFrom_s3CopyObjectInput_s3PutObjectInput copies matching elements from a to b
func setFrom_s3CopyObjectInput_s3PutObjectInput(a *s3.CopyObjectInput, b *s3.PutObjectInput) {
a.ACL = b.ACL
a.Bucket = b.Bucket
a.Key = b.Key
a.ACL = b.ACL
a.BucketKeyEnabled = b.BucketKeyEnabled
a.CacheControl = b.CacheControl
a.ChecksumAlgorithm = b.ChecksumAlgorithm
@@ -261,7 +270,6 @@ func setFrom_s3CopyObjectInput_s3PutObjectInput(a *s3.CopyObjectInput, b *s3.Put
a.GrantRead = b.GrantRead
a.GrantReadACP = b.GrantReadACP
a.GrantWriteACP = b.GrantWriteACP
a.Key = b.Key
a.Metadata = b.Metadata
a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
a.ObjectLockMode = b.ObjectLockMode

View File

@@ -3,6 +3,7 @@
package s3
import (
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
@@ -10,6 +11,9 @@ import (
"sort"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
v4signer "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
)
// URL parameters that need to be added to the signature
@@ -36,10 +40,17 @@ var s3ParamsToSign = map[string]struct{}{
"response-content-encoding": {},
}
// sign signs requests using v2 auth
// Implement HTTPSignerV4 interface
type v2Signer struct {
opt *Options
}
// SignHTTP signs requests using v2 auth.
//
// Cobbled together from goamz and aws-sdk-go
func sign(AccessKey, SecretKey string, req *http.Request) {
// Cobbled together from goamz and aws-sdk-go.
//
// Bodged up to compile with AWS SDK v2
func (v2 *v2Signer) SignHTTP(ctx context.Context, credentials aws.Credentials, req *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*v4signer.SignerOptions)) error {
// Set date
date := time.Now().UTC().Format(time.RFC1123)
req.Header.Set("Date", date)
@@ -107,11 +118,12 @@ func sign(AccessKey, SecretKey string, req *http.Request) {
// Make signature
payload := req.Method + "\n" + md5 + "\n" + contentType + "\n" + date + "\n" + joinedHeadersToSign + uri
hash := hmac.New(sha1.New, []byte(SecretKey))
hash := hmac.New(sha1.New, []byte(v2.opt.SecretAccessKey))
_, _ = hash.Write([]byte(payload))
signature := make([]byte, base64.StdEncoding.EncodedLen(hash.Size()))
base64.StdEncoding.Encode(signature, hash.Sum(nil))
// Set signature in request
req.Header.Set("Authorization", "AWS "+AccessKey+":"+string(signature))
req.Header.Set("Authorization", "AWS "+v2.opt.AccessKeyID+":"+string(signature))
return nil
}