GCS upload supports timestamps

CHANGELOG.md

@@ -1,3 +1,7 @@
+## v8.39.1 (July 7th 2025)
+### Implementation changes and bug fixes
+- [PR #2147](https://github.com/rqlite/rqlite/pull/2147): Support timestamped uploads to Google Cloud Storage.
+
 ## v8.39.0 (July 6th 2025)
 ### New features
 - [PR #2144](https://github.com/rqlite/rqlite/pull/2144), [PR #2145](https://github.com/rqlite/rqlite/pull/2145), [PR #2146](https://github.com/rqlite/rqlite/pull/2146): Automatic _Backup and Restore_ now supports Google Cloud Storage.

auto/backup/config.go

@@ -41,19 +41,22 @@ func NewStorageClient(data []byte) (*Config, StorageClient, error) {
 		if err != nil {
 			return nil, nil, err
 		}
-		s3ClientOps := &aws.S3ClientOpts{
+		opts := &aws.S3ClientOpts{
 			ForcePathStyle: s3cfg.ForcePathStyle,
 			Timestamp:      cfg.Timestamp,
 		}
 		sc, err = aws.NewS3Client(s3cfg.Endpoint, s3cfg.Region, s3cfg.AccessKeyID, s3cfg.SecretAccessKey,
-			s3cfg.Bucket, s3cfg.Path, s3ClientOps)
+			s3cfg.Bucket, s3cfg.Path, opts)
 	case auto.StorageTypeGCS:
 		gcsCfg := &gcp.GCSConfig{}
 		err = json.Unmarshal(cfg.Sub, gcsCfg)
 		if err != nil {
 			return nil, nil, err
 		}
-		sc, err = gcp.NewGCSClient(gcsCfg)
+		opts := &gcp.GCSClientOpts{
+			Timestamp: cfg.Timestamp,
+		}
+		sc, err = gcp.NewGCSClient(gcsCfg, opts)
 	default:
 		return nil, nil, auto.ErrUnsupportedStorageType
 	}

auto/backup/config_test.go

@@ -295,7 +295,7 @@ func mustNewGCSClient(t *testing.T, bucket, name, projectID, credentialsFile str
 		Name:            name,
 		ProjectID:       projectID,
 		CredentialsPath: credentialsFile,
-	})
+	}, nil)
 	if err != nil {
 		t.Fatalf("Failed to create GCS client: %v", err)
 	}

auto/restore/config.go

@@ -73,7 +73,7 @@ func NewStorageClient(data []byte) (*Config, StorageClient, error) {
 		if err != nil {
 			return nil, nil, err
 		}
-		sc, err = gcp.NewGCSClient(gcsCfg)
+		sc, err = gcp.NewGCSClient(gcsCfg, nil)
 	default:
 		return nil, nil, auto.ErrUnsupportedStorageType
 	}

gcp/example/main.go

@@ -20,7 +20,7 @@ func main() {
 		CredentialsPath: os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"),
 	}
 
-	client, err := gcp.NewGCSClient(&cfg)
+	client, err := gcp.NewGCSClient(&cfg, nil)
 	if err != nil {
 		panic(err)
 	}

gcp/gcs.go

@@ -28,6 +28,14 @@ var (
 	jwtAudTarget            = "https://oauth2.googleapis.com/token"
 )
 
+// TimestampedPath returns a new path with the given timestamp prepended.
+// If path contains /, the timestamp is prepended to the last segment.
+func TimestampedPath(path string, t time.Time) string {
+	parts := strings.Split(path, "/")
+	parts[len(parts)-1] = fmt.Sprintf("%s_%s", t.Format("20060102150405"), parts[len(parts)-1])
+	return strings.Join(parts, "/")
+}
+
 // GCSConfig is the subconfig for the GCS storage type.
 type GCSConfig struct {
 	Endpoint        string `json:"endpoint,omitempty"`
@@ -50,10 +58,17 @@ type GCSClient struct {
 	uploadURL string
 	objectURL string
 	bucketURL string
+
+	timestamp bool
+}
+
+// GCSClientOpts are options for creating a GCSClient.
+type GCSClientOpts struct {
+	Timestamp bool
 }
 
 // NewGCSClient returns an instance of a GCSClient.
-func NewGCSClient(cfg *GCSConfig) (*GCSClient, error) {
+func NewGCSClient(cfg *GCSConfig, opts *GCSClientOpts) (*GCSClient, error) {
 	if cfg.Endpoint == "" {
 		cfg.Endpoint = defaultEndpoint
 	}
@@ -71,6 +86,7 @@ func NewGCSClient(cfg *GCSConfig) (*GCSClient, error) {
 		objectURL: fmt.Sprintf("%s/storage/v1/b/%s/o/%s",
 			base, url.PathEscape(cfg.Bucket), url.PathEscape(cfg.Name)),
 		bucketURL: fmt.Sprintf("%s/storage/v1/b/%s", base, url.PathEscape(cfg.Bucket)),
+		timestamp: opts != nil && opts.Timestamp,
 	}, nil
 }
 
@@ -126,13 +142,17 @@ func (g *GCSClient) Upload(ctx context.Context, r io.Reader, id string) error {
 	var buf bytes.Buffer
 	w := multipart.NewWriter(&buf)
 
+	name := g.cfg.Name
+	if g.timestamp {
+		name = TimestampedPath(name, time.Now().UTC())
+	}
 	metaData := struct {
 		Name     string `json:"name"`
 		Metadata struct {
 			ID string `json:"id"`
 		} `json:"metadata"`
 	}{
-		Name: g.cfg.Name,
+		Name: name,
 	}
 	metaData.Metadata.ID = id
 

gcp/gcs_test.go

@@ -26,7 +26,7 @@ func Test_NewGCSClient(t *testing.T) {
 		CredentialsPath: createCredFile(t), // dummy, never used
 	}
 
-	cli, err := NewGCSClient(cfg)
+	cli, err := NewGCSClient(cfg, nil)
 	if err != nil {
 		t.Fatalf("NewGCSClient: %v", err)
 	}
@@ -112,7 +112,7 @@ func Test_Upload(t *testing.T) {
 			t.Fatalf("method %s", r.Method)
 		}
 		if !strings.HasPrefix(r.URL.Path, "/upload/storage/v1/b/mybucket/o") {
-			t.Fatalf("path %s", r.URL.Path)
+			t.Fatalf("received path does not have correct prefix: %s", r.URL.Path)
 		}
 		if r.Header.Get("Authorization") != "Bearer TESTTOKEN" {
 			t.Fatalf("auth header missing")
@@ -261,7 +261,7 @@ func newTestClient(t *testing.T, h http.HandlerFunc) (*GCSClient, func()) {
 		CredentialsPath: createCredFile(t), // dummy, never used
 	}
 
-	cli, err := NewGCSClient(cfg)
+	cli, err := NewGCSClient(cfg, nil)
 	if err != nil {
 		t.Fatalf("NewGCSClient: %v", err)
 	}