Upgrade to AWS SDK v2

Fixes https://github.com/rqlite/rqlite/issues/1655.
2026-01-25 04:16:26 +00:00 · 2024-08-29 08:30:57 -04:00
parent 270e259b84
commit bbc23ac633
6 changed files with 227 additions and 89 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,7 @@
 ## v8.29.3 (unreleased)
 ### Implementation changes and bug fixes
 - [PR #1872](https://github.com/rqlite/rqlite/pull/1872): Upgrade Hashicorp Raft to v1.7.1.
+- [PR #1873](https://github.com/rqlite/rqlite/pull/1873): Upgrade to AWS SDK v2. Fixes issue [#1655](https://github.com/rqlite/rqlite/issues/1655).

 ## v8.29.2 (August 26th 2024)
 There are no functional changes to rqlite in this release, but it does improve the rqlite Docker images.
--- a/aws/s3.go
+++ b/aws/s3.go
@@ -4,19 +4,18 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"net/http"
 	"strings"
 	"time"

-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 )

 var (
-	AWSS3IDKey = http.CanonicalHeaderKey("x-rqlite-auto-backup-id")
+	AWSS3IDKey = "x-rqlite-auto-backup-id"
 )

 // S3Config is the subconfig for the S3 storage type
@@ -32,17 +31,15 @@ type S3Config struct {

 // S3Client is a client for uploading data to S3.
 type S3Client struct {
-	endpoint       string
-	region         string
-	accessKey      string
-	secretKey      string
-	bucket         string
-	key            string
-	forcePathStyle bool
-	timestamp      bool
+	endpoint  string
+	region    string
+	accessKey string
+	secretKey string
+	bucket    string
+	key       string
+	timestamp bool

-	session *session.Session
-	s3      *s3.S3
+	s3 *s3.Client

 	// These fields are used for testing via dependency injection.
 	uploader   uploader
@@ -58,24 +55,28 @@ type S3ClientOpts struct {

 // NewS3Client returns an instance of an S3Client. opts can be nil.
 func NewS3Client(endpoint, region, accessKey, secretKey, bucket, key string, opts *S3ClientOpts) (*S3Client, error) {
-	cfg := aws.Config{
-		Endpoint: aws.String(endpoint),
-		Region:   aws.String(region),
-	}
-	if opts != nil {
-		cfg.S3ForcePathStyle = aws.Bool(opts.ForcePathStyle)
-	}
-	// If credentials aren't provided by the user, the AWS SDK will use the default
-	// credential provider chain, which supports environment variables, shared credentials
-	// file, and EC2 instance roles.
-	if accessKey != "" && secretKey != "" {
-		cfg.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "")
-	}
-	sess, err := session.NewSession(&cfg)
+	// Load the default config
+	cfg, err := config.LoadDefaultConfig(context.Background(),
+		config.WithRegion(region),
+	)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("unable to load SDK config, %v", err)
 	}
-	s3 := s3.New(sess)
+
+	// If credentials are provided, set them
+	if accessKey != "" && secretKey != "" {
+		cfg.Credentials = aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""))
+	}
+
+	// If an endpoint is provided, set it and the path style
+	s3 := s3.NewFromConfig(cfg, func(o *s3.Options) {
+		if opts != nil {
+			if endpoint != "" {
+				o.BaseEndpoint = aws.String(endpoint)
+			}
+			o.UsePathStyle = opts.ForcePathStyle
+		}
+	})

 	client := &S3Client{
 		endpoint:  endpoint,
@@ -85,14 +86,11 @@ func NewS3Client(endpoint, region, accessKey, secretKey, bucket, key string, opt
 		bucket:    bucket,
 		key:       key,

-		session: sess,
-		s3:      s3,
-
-		uploader:   s3manager.NewUploaderWithClient(s3),
-		downloader: s3manager.NewDownloaderWithClient(s3),
+		s3:         s3,
+		uploader:   manager.NewUploader(s3),
+		downloader: manager.NewDownloader(s3),
 	}
 	if opts != nil {
-		client.forcePathStyle = opts.ForcePathStyle
 		client.timestamp = opts.Timestamp
 	}
 	return client, nil
@@ -103,7 +101,7 @@ func (s *S3Client) String() string {
 	if s.endpoint == "" || strings.HasSuffix(s.endpoint, "amazonaws.com") {
 		// Native Amazon S3, use AWS's S3 URL format
 		return fmt.Sprintf("s3://%s/%s", s.bucket, s.key)
-	} else if !s.forcePathStyle {
+	} else if !s.s3.Options().UsePathStyle {
 		// Endpoint specified but not using path style (e.g. Wasabi)
 		return fmt.Sprintf("s3://%s.%s/%s", s.bucket, s.endpoint, s.key)
 	}
@@ -122,18 +120,18 @@ func (s *S3Client) Upload(ctx context.Context, reader io.Reader, id string) erro
 		}
 		key = TimestampedPath(key, s.now())
 	}
-	input := &s3manager.UploadInput{
+	input := &s3.PutObjectInput{
 		Bucket: aws.String(s.bucket),
 		Key:    aws.String(key),
 		Body:   reader,
 	}

 	if id != "" {
-		input.Metadata = map[string]*string{
-			AWSS3IDKey: aws.String(id),
+		input.Metadata = map[string]string{
+			AWSS3IDKey: id,
 		}
 	}
-	_, err := s.uploader.UploadWithContext(ctx, input)
+	_, err := s.uploader.Upload(ctx, input)
 	if err != nil {
 		return fmt.Errorf("failed to upload to %v: %w", s, err)
 	}
@@ -147,7 +145,7 @@ func (s *S3Client) CurrentID(ctx context.Context) (string, error) {
 		Key:    aws.String(s.key),
 	}

-	result, err := s.s3.HeadObjectWithContext(ctx, input)
+	result, err := s.s3.HeadObject(ctx, input)
 	if err != nil {
 		return "", fmt.Errorf("failed to get object head for %v: %w", s, err)
 	}
@@ -156,12 +154,12 @@ func (s *S3Client) CurrentID(ctx context.Context) (string, error) {
 	if !ok {
 		return "", fmt.Errorf("sum metadata not found for %v", s)
 	}
-	return *id, nil
+	return id, nil
 }

 // Download downloads data from S3.
 func (s *S3Client) Download(ctx context.Context, writer io.WriterAt) error {
-	_, err := s.downloader.DownloadWithContext(ctx, writer, &s3.GetObjectInput{
+	_, err := s.downloader.Download(ctx, writer, &s3.GetObjectInput{
 		Bucket: aws.String(s.bucket),
 		Key:    aws.String(s.key),
 	})
@@ -171,6 +169,18 @@ func (s *S3Client) Download(ctx context.Context, writer io.WriterAt) error {
 	return nil
 }

+// Delete deletes object from S3.
+func (s *S3Client) Delete(ctx context.Context) error {
+	_, err := s.s3.DeleteObject(ctx, &s3.DeleteObjectInput{
+		Bucket: aws.String(s.bucket),
+		Key:    aws.String(s.key),
+	})
+	if err != nil {
+		return fmt.Errorf("failed to delete %v: %w", s, err)
+	}
+	return nil
+}
+
 // TimestampedPath returns a new path with the given timestamp prepended.
 // If path contains /, the timestamp is prepended to the last segment.
 func TimestampedPath(path string, t time.Time) string {
@@ -180,9 +190,9 @@ func TimestampedPath(path string, t time.Time) string {
 }

 type uploader interface {
-	UploadWithContext(ctx aws.Context, input *s3manager.UploadInput, opts ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error)
+	Upload(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error)
 }

 type downloader interface {
-	DownloadWithContext(ctx aws.Context, w io.WriterAt, input *s3.GetObjectInput, opts ...func(*s3manager.Downloader)) (n int64, err error)
+	Download(ctx context.Context, w io.WriterAt, input *s3.GetObjectInput, opts ...func(*manager.Downloader)) (n int64, err error)
 }
--- a/aws/s3_test.go
+++ b/aws/s3_test.go
@@ -3,16 +3,19 @@ package aws
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
-	"net/http"
+	"os"
 	"strings"
 	"testing"
 	"time"

-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/rqlite/rqlite/v8/random"
 )

 func Test_NewS3Client(t *testing.T) {
@@ -35,8 +38,8 @@ func Test_NewS3Client(t *testing.T) {
 	if c.key != "key3" {
 		t.Fatalf("expected key to be %q, got %q", "key3", c.key)
 	}
-	if c.forcePathStyle != true {
-		t.Fatalf("expected forcePathStyle to be %v, got %v", true, c.forcePathStyle)
+	if c.s3.Options().UsePathStyle != true {
+		t.Fatalf("expected forcePathStyle to be %v, got %v", true, c.s3.Options().UsePathStyle)
 	}
 }

@@ -86,7 +89,7 @@ func Test_S3ClientUploadOK(t *testing.T) {
 	uploadedData := new(bytes.Buffer)

 	mockUploader := &mockUploader{
-		uploadFn: func(ctx aws.Context, input *s3manager.UploadInput, opts ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) {
+		uploadFn: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
 			if *input.Bucket != bucket {
 				t.Errorf("expected bucket to be %q, got %q", bucket, *input.Bucket)
 			}
@@ -103,11 +106,11 @@ func Test_S3ClientUploadOK(t *testing.T) {
 			if input.Metadata == nil {
 				t.Errorf("expected metadata to be non-nil")
 			}
-			exp, got := "some-id", *input.Metadata[http.CanonicalHeaderKey(AWSS3IDKey)]
+			exp, got := "some-id", input.Metadata[AWSS3IDKey]
 			if exp != got {
 				t.Errorf("expected metadata to contain %q, got %q", exp, got)
 			}
-			return &s3manager.UploadOutput{}, nil
+			return &manager.UploadOutput{}, nil
 		},
 	}

@@ -141,9 +144,8 @@ func Test_S3ClientUploadOK_Timestamped(t *testing.T) {
 	timestampedKey := "your/key/20210701150405_path"
 	expectedData := "test data"
 	uploadedData := new(bytes.Buffer)
-
 	mockUploader := &mockUploader{
-		uploadFn: func(ctx aws.Context, input *s3manager.UploadInput, opts ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) {
+		uploadFn: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
 			if *input.Bucket != bucket {
 				t.Errorf("expected bucket to be %q, got %q", bucket, *input.Bucket)
 			}
@@ -160,11 +162,11 @@ func Test_S3ClientUploadOK_Timestamped(t *testing.T) {
 			if input.Metadata == nil {
 				t.Errorf("expected metadata to be non-nil")
 			}
-			exp, got := "some-id", *input.Metadata[http.CanonicalHeaderKey(AWSS3IDKey)]
+			exp, got := "some-id", input.Metadata[AWSS3IDKey]
 			if exp != got {
 				t.Errorf("expected metadata to contain %q, got %q", exp, got)
 			}
-			return &s3manager.UploadOutput{}, nil
+			return &manager.UploadOutput{}, nil
 		},
 	}

@@ -204,7 +206,7 @@ func Test_S3ClientUploadOK_Timestamped_Changes(t *testing.T) {

 	timestampedKey1 := ""
 	mockUploader := &mockUploader{
-		uploadFn: func(ctx aws.Context, input *s3manager.UploadInput, opts ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) {
+		uploadFn: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
 			if timestampedKey1 == "" {
 				timestampedKey1 = *input.Key
 			} else {
@@ -212,7 +214,7 @@ func Test_S3ClientUploadOK_Timestamped_Changes(t *testing.T) {
 					t.Errorf("expected key for second upload to be different from %q, got %q", timestampedKey1, *input.Key)
 				}
 			}
-			return &s3manager.UploadOutput{}, nil
+			return &manager.UploadOutput{}, nil
 		},
 	}

@@ -252,7 +254,7 @@ func Test_S3ClientUploadOK_Timestamped_NoChanges(t *testing.T) {

 	timestampedKey1 := ""
 	mockUploader := &mockUploader{
-		uploadFn: func(ctx aws.Context, input *s3manager.UploadInput, opts ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) {
+		uploadFn: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
 			if timestampedKey1 == "" {
 				timestampedKey1 = *input.Key
 			} else {
@@ -260,7 +262,7 @@ func Test_S3ClientUploadOK_Timestamped_NoChanges(t *testing.T) {
 					t.Errorf("expected key for second upload to be same as %q, got %q", timestampedKey1, *input.Key)
 				}
 			}
-			return &s3manager.UploadOutput{}, nil
+			return &manager.UploadOutput{}, nil
 		},
 	}

@@ -296,8 +298,8 @@ func Test_S3ClientUploadFail(t *testing.T) {
 	key := "your/key/path"

 	mockUploader := &mockUploader{
-		uploadFn: func(ctx aws.Context, input *s3manager.UploadInput, opts ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) {
-			return &s3manager.UploadOutput{}, fmt.Errorf("some error related to S3")
+		uploadFn: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
+			return &manager.UploadOutput{}, fmt.Errorf("some error related to S3")
 		},
 	}

@@ -329,7 +331,7 @@ func Test_S3ClientDownloadOK(t *testing.T) {
 	expectedData := "test data"

 	mockDownloader := &mockDownloader{
-		downloadFn: func(ctx aws.Context, w io.WriterAt, input *s3.GetObjectInput, opts ...func(*s3manager.Downloader)) (int64, error) {
+		downloadFn: func(ctx context.Context, w io.WriterAt, input *s3.GetObjectInput, opts ...func(*manager.Downloader)) (int64, error) {
 			if *input.Bucket != bucket {
 				t.Errorf("expected bucket to be %q, got %q", bucket, *input.Bucket)
 			}
@@ -353,7 +355,7 @@ func Test_S3ClientDownloadOK(t *testing.T) {
 		downloader: mockDownloader,
 	}

-	writer := aws.NewWriteAtBuffer(make([]byte, len(expectedData)))
+	writer := manager.NewWriteAtBuffer(make([]byte, 0, len(expectedData)))
 	err := client.Download(context.Background(), writer)
 	if err != nil {
 		t.Fatalf("Unexpected error: %v", err)
@@ -372,7 +374,7 @@ func Test_S3ClientDownloadFail(t *testing.T) {
 	key := "your/key/path"

 	mockDownloader := &mockDownloader{
-		downloadFn: func(ctx aws.Context, w io.WriterAt, input *s3.GetObjectInput, opts ...func(*s3manager.Downloader)) (n int64, err error) {
+		downloadFn: func(ctx context.Context, w io.WriterAt, input *s3.GetObjectInput, opts ...func(*manager.Downloader)) (n int64, err error) {
 			return 0, fmt.Errorf("some error related to S3")
 		},
 	}
@@ -387,7 +389,7 @@ func Test_S3ClientDownloadFail(t *testing.T) {
 		downloader: mockDownloader,
 	}

-	writer := aws.NewWriteAtBuffer(nil)
+	writer := manager.NewWriteAtBuffer(nil)
 	err := client.Download(context.Background(), writer)
 	if err == nil {
 		t.Fatal("Expected error, got nil")
@@ -397,6 +399,85 @@ func Test_S3ClientDownloadFail(t *testing.T) {
 	}
 }

+func Test_CreateDeleteOK(t *testing.T) {
+	exists := func(client *s3.Client, bucket, key string) bool {
+		t.Helper()
+		input := &s3.HeadObjectInput{
+			Bucket: aws.String(bucket),
+			Key:    aws.String(key),
+		}
+		_, err := client.HeadObject(context.TODO(), input)
+		if err != nil {
+			var notFound *types.NotFound
+			if errors.As(err, &notFound) {
+				return false
+			}
+			t.Fatalf("error checking if object exists: %v", err)
+		}
+		return true
+	}
+
+	accesskey, ok := os.LookupEnv("RQLITE_S3_ACCESS_KEY")
+	if !ok {
+		t.Skip("RQLITE_S3_ACCESS_KEY is not set")
+	}
+	secretKey, ok := os.LookupEnv("RQLITE_S3_SECRET_ACCESS_KEY")
+	if !ok {
+		t.Skip("RQLITE_S3_SECRET_ACCESS_KEY is not set")
+	}
+
+	path := random.String()
+	client, err := NewS3Client("s3.amazonaws.com", "us-west-2", accesskey, secretKey, "rqlite-testing-circleci", path, nil)
+	if err != nil {
+		t.Fatalf("error creating S3 client: %v", err)
+	}
+	if err := client.Upload(context.Background(), strings.NewReader("test data"), "the-id"); err != nil {
+		t.Fatalf("error uploading to S3: %v", err)
+	}
+	if !exists(client.s3, client.bucket, client.key) {
+		t.Fatalf("expected object to exist")
+	}
+
+	if err := client.Delete(context.Background()); err != nil {
+		t.Fatalf("error deleting from S3: %v", err)
+	}
+	if exists(client.s3, client.bucket, client.key) {
+		t.Fatalf("expected object to not exist")
+	}
+}
+
+func Test_MetadataSet(t *testing.T) {
+	accesskey, ok := os.LookupEnv("RQLITE_S3_ACCESS_KEY")
+	if !ok {
+		t.Skip("RQLITE_S3_ACCESS_KEY is not set")
+	}
+	secretKey, ok := os.LookupEnv("RQLITE_S3_SECRET_ACCESS_KEY")
+	if !ok {
+		t.Skip("RQLITE_S3_SECRET_ACCESS_KEY is not set")
+	}
+
+	path := random.String()
+	client, err := NewS3Client("s3.amazonaws.com", "us-west-2", accesskey, secretKey, "rqlite-testing-circleci", path, nil)
+	if err != nil {
+		t.Fatalf("error creating S3 client: %v", err)
+	}
+	if err := client.Upload(context.Background(), strings.NewReader("test data"), "the-id"); err != nil {
+		t.Fatalf("error uploading to S3: %v", err)
+	}
+	defer func() {
+		client.Delete(context.Background())
+	}()
+
+	// Now check that the metadata is set
+	id, err := client.CurrentID(context.Background())
+	if err != nil {
+		t.Fatalf("error getting current ID: %v", err)
+	}
+	if id != "the-id" {
+		t.Fatalf("expected ID to be %q, got %q", "the-id", id)
+	}
+}
+
 func Test_TimestampedPath(t *testing.T) {
 	ts, err := time.Parse(time.RFC3339, "2021-07-01T15:04:05Z")
 	if err != nil {
@@ -427,10 +508,10 @@ func Test_TimestampedPath_Changes(t *testing.T) {
 }

 type mockDownloader struct {
-	downloadFn func(ctx aws.Context, w io.WriterAt, input *s3.GetObjectInput, opts ...func(*s3manager.Downloader)) (n int64, err error)
+	downloadFn func(ctx context.Context, w io.WriterAt, input *s3.GetObjectInput, opts ...func(*manager.Downloader)) (n int64, err error)
 }

-func (m *mockDownloader) DownloadWithContext(ctx aws.Context, w io.WriterAt, input *s3.GetObjectInput, opts ...func(*s3manager.Downloader)) (n int64, err error) {
+func (m *mockDownloader) Download(ctx context.Context, w io.WriterAt, input *s3.GetObjectInput, opts ...func(*manager.Downloader)) (n int64, err error) {
 	if m.downloadFn != nil {
 		return m.downloadFn(ctx, w, input, opts...)
 	}
@@ -438,14 +519,14 @@ func (m *mockDownloader) DownloadWithContext(ctx aws.Context, w io.WriterAt, inp
 }

 type mockUploader struct {
-	uploadFn func(ctx aws.Context, input *s3manager.UploadInput, opts ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error)
+	uploadFn func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error)
 }

-func (m *mockUploader) UploadWithContext(ctx aws.Context, input *s3manager.UploadInput, opts ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) {
+func (m *mockUploader) Upload(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
 	if m.uploadFn != nil {
 		return m.uploadFn(ctx, input, opts...)
 	}
-	return &s3manager.UploadOutput{}, nil
+	return &manager.UploadOutput{}, nil
 }

 func forcePathStyleOptions() *S3ClientOpts {
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,11 @@ go 1.21

 require (
 	github.com/Bowery/prompt v0.0.0-20190916142128-fa8279994f75
-	github.com/aws/aws-sdk-go v1.55.5
+	github.com/aws/aws-sdk-go-v2 v1.30.4
+	github.com/aws/aws-sdk-go-v2/config v1.27.31
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.30
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.16
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.61.0
 	github.com/hashicorp/go-hclog v1.6.3
 	github.com/hashicorp/raft v1.7.1
 	github.com/mkideal/cli v0.2.7
@@ -20,6 +24,20 @@ require (

 require (
 	github.com/armon/go-metrics v0.5.3 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 // indirect
+	github.com/aws/smithy-go v1.20.4 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/fatih/color v1.17.0 // indirect
@@ -35,7 +53,6 @@ require (
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/hashicorp/serf v0.10.1 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
--- a/go.sum
+++ b/go.sum
@@ -8,8 +8,44 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
-github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8=
+github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw=
+github.com/aws/aws-sdk-go-v2/config v1.27.31 h1:kxBoRsjhT3pq0cKthgj6RU6bXTm/2SgdoUMyrVw0rAI=
+github.com/aws/aws-sdk-go-v2/config v1.27.31/go.mod h1:z04nZdSWFPaDwK3DdJOG2r+scLQzMYuJeW0CujEm9FM=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.30 h1:aau/oYFtibVovr2rDt8FHlU17BTicFEMAi29V1U+L5Q=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.30/go.mod h1:BPJ/yXV92ZVq6G8uYvbU0gSl8q94UB63nMT5ctNO38g=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 h1:yjwoSyDZF8Jth+mUk5lSPJCkMC0lMy6FaCD51jm6ayE=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12/go.mod h1:fuR57fAgMk7ot3WcNQfb6rSEn+SUffl7ri+aa8uKysI=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.16 h1:1FWqcOnvnO0lRsv0kLACwwQquoZIoS5tD0MtfoNdnkk=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.16/go.mod h1:+E8OuB446P/5Swajo40TqenLMzm6aYDEEz6FZDn/u1E=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 h1:mimdLQkIX1zr8GIPY1ZtALdBQGxcASiBd2MOp8m/dMc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16/go.mod h1:YHk6owoSwrIsok+cAH9PENCOGoH5PU2EllX4vLtSrsY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 h1:GckUnpm4EJOAio1c8o25a+b3lVfwVzC9gnSBqiiNmZM=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18/go.mod h1:Br6+bxfG33Dk3ynmkhsW2Z/t9D4+lRqdLDNCKi85w0U=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 h1:tJ5RnkHCiSH0jyd6gROjlJtNwov0eGYNz8s8nFcR0jQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18/go.mod h1:++NHzT+nAF7ZPrHPsA+ENvsXkOO8wEu+C6RXltAG4/c=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 h1:jg16PhLPUiHIj8zYIW6bqzeQSuHVEiWnGA0Brz5Xv2I=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16/go.mod h1:Uyk1zE1VVdsHSU7096h/rwnXDzOzYQVl+FNPhPw7ShY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.61.0 h1:Wb544Wh+xfSXqJ/j3R4aX9wrKUoZsJNmilBYZb3mKQ4=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.61.0/go.mod h1:BSPI0EfnYUuNHPS0uqIo5VrRwzie+Fp+YhQOUs16sKI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 h1:zCsFCKvbj25i7p1u94imVoO447I/sFv8qq+lGJhRN0c=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.5/go.mod h1:ZeDX1SnKsVlejeuz41GiajjZpRSWR7/42q/EyA/QEiM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 h1:SKvPgvdvmiTWoi0GAJ7AsJfOz3ngVkD/ERbs5pUnHNI=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5/go.mod h1:20sz31hv/WsPa3HhU3hfrIet2kxM4Pe0r20eBZ20Tac=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 h1:OMsEmCyz2i89XwRwPouAJvhj81wINh+4UK+k/0Yo/q8=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.5/go.mod h1:vmSqFK+BVIwVpDAGZB3CoCXHzurt4qBE8lf+I/kRTh0=
+github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
+github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
@@ -111,10 +147,6 @@ github.com/hashicorp/raft v1.7.1/go.mod h1:hUeiEwQQR/Nk2iKDD0dkEhklSsu3jcAcqvPzP
 github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I=
 github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
 github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
-github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
-github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@@ -334,8 +366,5 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
--- a/system_test/e2e/helpers.py
+++ b/system_test/e2e/helpers.py
@@ -449,7 +449,7 @@ class Node(object):
      return self.num_auto_backups()

    n = self.num_auto_backups()
-    raise Exception('rqlite node failed to idle backups within %d seconds (%s)' % n)
+    raise Exception('rqlite node failed to idle backups within %d seconds (%s)' % (timeout, n))

  def wait_for_fsm_index(self, index, timeout=TIMEOUT):
    '''