Ything-RepoMirrors/litestream
1
0
Fork 0
mirror of https://github.com/benbjohnson/litestream.git synced 2026-01-25 05:06:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: preserve LTX file timestamps during compaction and storage operations (#778)

Browse Source 
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Cory LaNou
2025-10-13 14:01:42 -05:00
committed by GitHub
parent 1b613e9f05
commit f236f5099f
16 changed files with 552 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
abs/replica_client.go
View File

@@ -1,6 +1,7 @@
package abs
import (
"bytes"
"context"
"errors"
"fmt"
@@ -30,6 +31,10 @@ import (
// ReplicaClientType is the client type for this package.
const ReplicaClientType = "abs"
// MetadataKeyTimestamp is the metadata key for storing LTX file timestamps in Azure Blob Storage.
// Azure metadata keys cannot contain hyphens, so we use litestreamtimestamp (C# identifier rules).
const MetadataKeyTimestamp = "litestreamtimestamp"
var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
// ReplicaClient is a client for writing LTX files to Azure Blob Storage.
@@ -145,7 +150,9 @@ func (c *ReplicaClient) Init(ctx context.Context) (err error) {
}
// LTXFiles returns an iterator over all available LTX files.
func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID) (ltx.FileIterator, error) {
// Azure always uses accurate timestamps from metadata since they're included in LIST operations at zero cost.
// The useMetadata parameter is ignored.
func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error) {
if err := c.Init(ctx); err != nil {
return nil, err
}
@@ -159,16 +166,31 @@ func (c *ReplicaClient) WriteLTXFile(ctx context.Context, level int, minTXID, ma
}
key := litestream.LTXFilePath(c.Path, level, minTXID, maxTXID)
startTime := time.Now()
rc := internal.NewReadCounter(rd)
// Use TeeReader to peek at LTX header while preserving data for upload
var buf bytes.Buffer
teeReader := io.TeeReader(rd, &buf)
// Upload blob with proper content type and access tier
// Extract timestamp from LTX header
hdr, _, err := ltx.PeekHeader(teeReader)
if err != nil {
return nil, fmt.Errorf("extract timestamp from LTX header: %w", err)
}
timestamp := time.UnixMilli(hdr.Timestamp).UTC()
// Combine buffered data with rest of reader
rc := internal.NewReadCounter(io.MultiReader(&buf, rd))
// Upload blob with proper content type, access tier, and metadata
// Azure metadata keys cannot contain hyphens, so use litestreamtimestamp
_, err = c.client.UploadStream(ctx, c.Bucket, key, rc, &azblob.UploadStreamOptions{
HTTPHeaders: &blob.HTTPHeaders{
BlobContentType: to.Ptr("application/octet-stream"),
},
AccessTier: to.Ptr(blob.AccessTierHot), // Use Hot tier as default
Metadata: map[string]*string{
MetadataKeyTimestamp: to.Ptr(timestamp.Format(time.RFC3339Nano)),
},
})
if err != nil {
return nil, fmt.Errorf("abs: cannot upload ltx file %q: %w", key, err)
@@ -182,7 +204,7 @@ func (c *ReplicaClient) WriteLTXFile(ctx context.Context, level int, minTXID, ma
MinTXID: minTXID,
MaxTXID: maxTXID,
Size: rc.N(),
CreatedAt: startTime.UTC(),
CreatedAt: timestamp,
}, nil
}
@@ -370,11 +392,10 @@ func (itr *ltxFileIterator) loadNextPage() bool {
// Build file info
info := &ltx.FileInfo{
Level: itr.level,
MinTXID: minTXID,
MaxTXID: maxTXID,
Size: *item.Properties.ContentLength,
CreatedAt: item.Properties.CreationTime.UTC(),
Level: itr.level,
MinTXID: minTXID,
MaxTXID: maxTXID,
Size: *item.Properties.ContentLength,
}
// Skip if below seek TXID
@@ -387,6 +408,17 @@ func (itr *ltxFileIterator) loadNextPage() bool {
continue
}
// Always use accurate timestamp from metadata since it's zero-cost
// Azure includes metadata in LIST operations, so no extra API call needed
info.CreatedAt = item.Properties.CreationTime.UTC()
if item.Metadata != nil {
if ts, ok := item.Metadata[MetadataKeyTimestamp]; ok && ts != nil {
if parsed, err := time.Parse(time.RFC3339Nano, *ts); err == nil {
info.CreatedAt = parsed
}
}
}
itr.pageItems = append(itr.pageItems, info)
}

3
cmd/litestream/ltx.go
View File

@@ -71,7 +71,8 @@ func (c *LTXCommand) Run(ctx context.Context, args []string) (err error) {
defer w.Flush()
fmt.Fprintln(w, "min_txid\tmax_txid\tsize\tcreated")
itr, err := r.Client.LTXFiles(ctx, 0, 0)
// Normal operation - use fast timestamps
itr, err := r.Client.LTXFiles(ctx, 0, 0, false)
if err != nil {
return err
}

9
db.go
View File

@@ -1398,7 +1398,8 @@ func (db *DB) Compact(ctx context.Context, dstLevel int) (*ltx.FileInfo, error)
seekTXID := prevMaxInfo.MaxTXID + 1
// Collect files after last compaction.
itr, err := db.Replica.Client.LTXFiles(ctx, srcLevel, seekTXID)
// Normal operation - use fast timestamps
itr, err := db.Replica.Client.LTXFiles(ctx, srcLevel, seekTXID, false)
if err != nil {
return nil, fmt.Errorf("source ltx files after %s: %w", seekTXID, err)
}
@@ -1505,7 +1506,8 @@ func (db *DB) Snapshot(ctx context.Context) (*ltx.FileInfo, error) {
func (db *DB) EnforceSnapshotRetention(ctx context.Context, timestamp time.Time) (minSnapshotTXID ltx.TXID, err error) {
db.Logger.Debug("enforcing snapshot retention", "timestamp", timestamp)
itr, err := db.Replica.Client.LTXFiles(ctx, SnapshotLevel, 0)
// Normal operation - use fast timestamps
itr, err := db.Replica.Client.LTXFiles(ctx, SnapshotLevel, 0, false)
if err != nil {
return 0, fmt.Errorf("fetch ltx files: %w", err)
}
@@ -1551,7 +1553,8 @@ func (db *DB) EnforceSnapshotRetention(ctx context.Context, timestamp time.Time)
func (db *DB) EnforceRetentionByTXID(ctx context.Context, level int, txID ltx.TXID) (err error) {
db.Logger.Debug("enforcing retention", "level", level, "txid", txID)
itr, err := db.Replica.Client.LTXFiles(ctx, level, 0)
// Normal operation - use fast timestamps
itr, err := db.Replica.Client.LTXFiles(ctx, level, 0, false)
if err != nil {
return fmt.Errorf("fetch ltx files: %w", err)
}

120
db_test.go
View File

@@ -2,8 +2,10 @@ package litestream_test
import (
"context"
"fmt"
"hash/crc64"
"os"
"path/filepath"
"sync"
"testing"
"time"
@@ -11,6 +13,7 @@ import (
"github.com/superfly/ltx"
"github.com/benbjohnson/litestream"
"github.com/benbjohnson/litestream/file"
"github.com/benbjohnson/litestream/internal/testingutil"
)
@@ -522,7 +525,7 @@ func TestDB_EnforceRetention(t *testing.T) {
}
// Get list of snapshots before retention
itr, err := db.Replica.Client.LTXFiles(t.Context(), litestream.SnapshotLevel, 0)
itr, err := db.Replica.Client.LTXFiles(t.Context(), litestream.SnapshotLevel, 0, false)
if err != nil {
t.Fatal(err)
}
@@ -545,7 +548,7 @@ func TestDB_EnforceRetention(t *testing.T) {
}
// Verify snapshots after retention
itr, err = db.Replica.Client.LTXFiles(t.Context(), litestream.SnapshotLevel, 0)
itr, err = db.Replica.Client.LTXFiles(t.Context(), litestream.SnapshotLevel, 0, false)
if err != nil {
t.Fatal(err)
}
@@ -641,3 +644,116 @@ func TestDB_ConcurrentMapWrite(t *testing.T) {
t.Log("Test completed without race condition")
}
// TestCompaction_PreservesLastTimestamp verifies that after compaction,
// the resulting file's timestamp reflects the last source file timestamp
// as recorded in the LTX headers. This ensures point-in-time restoration
// continues to work after compaction (issue #771).
func TestCompaction_PreservesLastTimestamp(t *testing.T) {
ctx := context.Background()
db, sqldb := testingutil.MustOpenDBs(t)
defer testingutil.MustCloseDBs(t, db, sqldb)
// Set up replica with file backend
replicaPath := filepath.Join(t.TempDir(), "replica")
client := file.NewReplicaClient(replicaPath)
db.Replica = litestream.NewReplicaWithClient(db, client)
db.Replica.MonitorEnabled = false
// Create some transactions
for i := 0; i < 10; i++ {
if _, err := sqldb.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS t (id INTEGER PRIMARY KEY, val TEXT)`); err != nil {
t.Fatalf("create table: %v", err)
}
if _, err := sqldb.ExecContext(ctx, `INSERT INTO t (val) VALUES (?)`, fmt.Sprintf("value-%d", i)); err != nil {
t.Fatalf("insert %d: %v", i, err)
}
// Sync to create L0 files
if err := db.Sync(ctx); err != nil {
t.Fatalf("sync db: %v", err)
}
if err := db.Replica.Sync(ctx); err != nil {
t.Fatalf("sync replica: %v", err)
}
}
// Record the last L0 file timestamp before compaction
itr, err := client.LTXFiles(ctx, 0, 0, false)
if err != nil {
t.Fatalf("list L0 files: %v", err)
}
defer itr.Close()
l0Files, err := ltx.SliceFileIterator(itr)
if err != nil {
t.Fatalf("convert iterator: %v", err)
}
if err := itr.Close(); err != nil {
t.Fatalf("close iterator: %v", err)
}
var lastTime time.Time
for _, info := range l0Files {
if lastTime.IsZero() || info.CreatedAt.After(lastTime) {
lastTime = info.CreatedAt
}
}
if len(l0Files) == 0 {
t.Fatal("expected L0 files before compaction")
}
t.Logf("Found %d L0 files, last timestamp: %v", len(l0Files), lastTime)
// Perform compaction from L0 to L1
levels := litestream.CompactionLevels{
{Level: 0},
{Level: 1, Interval: time.Second},
}
store := litestream.NewStore([]*litestream.DB{db}, levels)
store.CompactionMonitorEnabled = false
if err := store.Open(ctx); err != nil {
t.Fatalf("open store: %v", err)
}
defer func() {
if err := store.Close(ctx); err != nil {
t.Fatalf("close store: %v", err)
}
}()
_, err = store.CompactDB(ctx, db, levels[1])
if err != nil {
t.Fatalf("compact: %v", err)
}
// Verify L1 file has the last timestamp from L0 files
itr, err = client.LTXFiles(ctx, 1, 0, false)
if err != nil {
t.Fatalf("list L1 files: %v", err)
}
defer itr.Close()
l1Files, err := ltx.SliceFileIterator(itr)
if err != nil {
t.Fatalf("convert L1 iterator: %v", err)
}
if err := itr.Close(); err != nil {
t.Fatalf("close L1 iterator: %v", err)
}
if len(l1Files) == 0 {
t.Fatal("expected L1 file after compaction")
}
l1Info := l1Files[0]
// The L1 file's CreatedAt should be the last timestamp from the L0 files
// Allow for some drift due to millisecond precision in LTX headers
timeDiff := l1Info.CreatedAt.Sub(lastTime)
if timeDiff.Abs() > time.Second {
t.Errorf("L1 CreatedAt = %v, last L0 = %v (diff: %v)", l1Info.CreatedAt, lastTime, timeDiff)
t.Error("L1 file timestamp should preserve last source file timestamp")
}
}

35
file/replica_client.go
View File

@@ -1,10 +1,13 @@
package file
import (
"bytes"
"context"
"fmt"
"io"
"os"
"path/filepath"
"time"
"github.com/superfly/ltx"
@@ -59,8 +62,9 @@ func (c *ReplicaClient) LTXFilePath(level int, minTXID, maxTXID ltx.TXID) string
return filepath.FromSlash(litestream.LTXFilePath(c.path, level, minTXID, maxTXID))
}
// LTXFiles returns an iterator over all available LTX files.
func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID) (ltx.FileIterator, error) {
// LTXFiles returns an iterator over all LTX files on the replica for the given level.
// The useMetadata parameter is ignored for file backend as ModTime is always available from readdir.
func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error) {
f, err := os.Open(c.LTXLevelDir(level))
if os.IsNotExist(err) {
return ltx.NewFileInfoSliceIterator(nil), nil
@@ -75,6 +79,7 @@ func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID)
}
// Iterate over every file and convert to metadata.
// ModTime contains the accurate timestamp set by Chtimes in WriteLTXFile.
infos := make([]*ltx.FileInfo, 0, len(fis))
for _, fi := range fis {
minTXID, maxTXID, err := ltx.ParseFilename(fi.Name())
@@ -117,13 +122,28 @@ func (c *ReplicaClient) OpenLTXFile(ctx context.Context, level int, minTXID, max
return f, nil
}
// WriteLTXFile writes an LTX file to disk.
// WriteLTXFile writes an LTX file to the replica.
// Extracts timestamp from LTX header and sets it as the file's ModTime to preserve original creation time.
func (c *ReplicaClient) WriteLTXFile(ctx context.Context, level int, minTXID, maxTXID ltx.TXID, rd io.Reader) (info *ltx.FileInfo, err error) {
var fileInfo, dirInfo os.FileInfo
if db := c.db(); db != nil {
fileInfo, dirInfo = db.FileInfo(), db.DirInfo()
}
// Use TeeReader to peek at LTX header while preserving data for upload
var buf bytes.Buffer
teeReader := io.TeeReader(rd, &buf)
// Extract timestamp from LTX header
hdr, _, err := ltx.PeekHeader(teeReader)
if err != nil {
return nil, fmt.Errorf("extract timestamp from LTX header: %w", err)
}
timestamp := time.UnixMilli(hdr.Timestamp).UTC()
// Combine buffered data with rest of reader
fullReader := io.MultiReader(&buf, rd)
// Ensure parent directory exists.
filename := c.LTXFilePath(level, minTXID, maxTXID)
if err := internal.MkdirAll(filepath.Dir(filename), dirInfo); err != nil {
@@ -137,7 +157,7 @@ func (c *ReplicaClient) WriteLTXFile(ctx context.Context, level int, minTXID, ma
}
defer f.Close()
if _, err := io.Copy(f, rd); err != nil {
if _, err := io.Copy(f, fullReader); err != nil {
return nil, err
}
if err := f.Sync(); err != nil {
@@ -154,7 +174,7 @@ func (c *ReplicaClient) WriteLTXFile(ctx context.Context, level int, minTXID, ma
MinTXID: minTXID,
MaxTXID: maxTXID,
Size: fi.Size(),
CreatedAt: fi.ModTime().UTC(),
CreatedAt: timestamp,
}
if err := f.Close(); err != nil {
@@ -166,6 +186,11 @@ func (c *ReplicaClient) WriteLTXFile(ctx context.Context, level int, minTXID, ma
return nil, err
}
// Set file ModTime to preserve original timestamp
if err := os.Chtimes(filename, timestamp, timestamp); err != nil {
return nil, err
}
return info, nil
}

64
gs/replica_client.go
View File

@@ -1,6 +1,7 @@
package gs
import (
"bytes"
"context"
"errors"
"fmt"
@@ -21,6 +22,9 @@ import (
// ReplicaClientType is the client type for this package.
const ReplicaClientType = "gs"
// MetadataKeyTimestamp is the metadata key for storing LTX file timestamps in GCS.
const MetadataKeyTimestamp = "litestream-timestamp"
var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
// ReplicaClient is a client for writing LTX files to Google Cloud Storage.
@@ -91,7 +95,9 @@ func (c *ReplicaClient) DeleteAll(ctx context.Context) error {
}
// LTXFiles returns an iterator over all available LTX files for a level.
func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID) (ltx.FileIterator, error) {
// GCS always uses accurate timestamps from metadata since they're included in LIST operations at zero cost.
// The useMetadata parameter is ignored.
func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error) {
if err := c.Init(ctx); err != nil {
return nil, err
}
@@ -102,7 +108,7 @@ func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID)
prefix += seek.String()
}
return newLTXFileIterator(c.bkt.Objects(ctx, &storage.Query{Prefix: prefix}), level), nil
return newLTXFileIterator(c.bkt.Objects(ctx, &storage.Query{Prefix: prefix}), c, level), nil
}
// WriteLTXFile writes an LTX file from rd to a remote path.
@@ -112,12 +118,30 @@ func (c *ReplicaClient) WriteLTXFile(ctx context.Context, level int, minTXID, ma
}
key := litestream.LTXFilePath(c.Path, level, minTXID, maxTXID)
startTime := time.Now()
// Use TeeReader to peek at LTX header while preserving data for upload
var buf bytes.Buffer
teeReader := io.TeeReader(rd, &buf)
// Extract timestamp from LTX header
hdr, _, err := ltx.PeekHeader(teeReader)
if err != nil {
return nil, fmt.Errorf("extract timestamp from LTX header: %w", err)
}
timestamp := time.UnixMilli(hdr.Timestamp).UTC()
// Combine buffered data with rest of reader
fullReader := io.MultiReader(&buf, rd)
w := c.bkt.Object(key).NewWriter(ctx)
defer w.Close()
n, err := io.Copy(w, rd)
// Store timestamp in GCS metadata for accurate timestamp retrieval
w.Metadata = map[string]string{
MetadataKeyTimestamp: timestamp.Format(time.RFC3339Nano),
}
n, err := io.Copy(w, fullReader)
if err != nil {
return info, err
} else if err := w.Close(); err != nil {
@@ -132,7 +156,7 @@ func (c *ReplicaClient) WriteLTXFile(ctx context.Context, level int, minTXID, ma
MinTXID: minTXID,
MaxTXID: maxTXID,
Size: n,
CreatedAt: startTime.UTC(),
CreatedAt: timestamp,
}, nil
}
@@ -176,16 +200,18 @@ func (c *ReplicaClient) DeleteLTXFiles(ctx context.Context, a []*ltx.FileInfo) e
}
type ltxFileIterator struct {
it *storage.ObjectIterator
level int
info *ltx.FileInfo
err error
it *storage.ObjectIterator
client *ReplicaClient
level int
info *ltx.FileInfo
err error
}
func newLTXFileIterator(it *storage.ObjectIterator, level int) *ltxFileIterator {
func newLTXFileIterator(it *storage.ObjectIterator, client *ReplicaClient, level int) *ltxFileIterator {
return &ltxFileIterator{
it: it,
level: level,
it: it,
client: client,
level: level,
}
}
@@ -215,14 +241,26 @@ func (itr *ltxFileIterator) Next() bool {
continue
}
// Always use accurate timestamp from metadata since it's zero-cost
// GCS includes metadata in LIST operations, so no extra API call needed
createdAt := attrs.Created.UTC()
if attrs.Metadata != nil {
if ts, ok := attrs.Metadata[MetadataKeyTimestamp]; ok {
if parsed, err := time.Parse(time.RFC3339Nano, ts); err == nil {
createdAt = parsed
}
}
}
// Store current snapshot and return.
itr.info = &ltx.FileInfo{
Level: itr.level,
MinTXID: minTXID,
MaxTXID: maxTXID,
Size: attrs.Size,
CreatedAt: attrs.Created.UTC(),
CreatedAt: createdAt,
}
return true
}
}

6
mock/replica_client.go
View File

@@ -13,7 +13,7 @@ var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
type ReplicaClient struct {
DeleteAllFunc func(ctx context.Context) error
LTXFilesFunc func(ctx context.Context, level int, seek ltx.TXID) (ltx.FileIterator, error)
LTXFilesFunc func(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error)
OpenLTXFileFunc func(ctx context.Context, level int, minTXID, maxTXID ltx.TXID, offset, size int64) (io.ReadCloser, error)
WriteLTXFileFunc func(ctx context.Context, level int, minTXID, maxTXID ltx.TXID, r io.Reader) (*ltx.FileInfo, error)
DeleteLTXFilesFunc func(ctx context.Context, a []*ltx.FileInfo) error
@@ -25,8 +25,8 @@ func (c *ReplicaClient) DeleteAll(ctx context.Context) error {
return c.DeleteAllFunc(ctx)
}
func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID) (ltx.FileIterator, error) {
return c.LTXFilesFunc(ctx, level, seek)
func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error) {
return c.LTXFilesFunc(ctx, level, seek, useMetadata)
}
func (c *ReplicaClient) OpenLTXFile(ctx context.Context, level int, minTXID, maxTXID ltx.TXID, offset, size int64) (io.ReadCloser, error) {

50
nats/replica_client.go
View File

@@ -1,6 +1,7 @@
package nats
import (
"bytes"
"context"
"errors"
"fmt"
@@ -23,6 +24,9 @@ import (
// ReplicaClientType is the client type for this package.
const ReplicaClientType = "nats"
// HeaderKeyTimestamp is the header key for storing LTX file timestamps in NATS object headers.
const HeaderKeyTimestamp = "Litestream-Timestamp"
var _ litestream.ReplicaClient = (*ReplicaClient)(nil)
// ReplicaClient is a client for writing LTX files to NATS JetStream Object Store.
@@ -230,7 +234,9 @@ func (c *ReplicaClient) parseLTXPath(objPath string) (level int, minTXID, maxTXI
}
// LTXFiles returns an iterator of all LTX files on the replica for a given level.
func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID) (ltx.FileIterator, error) {
// NATS always uses accurate timestamps from headers since they're included in LIST operations at zero cost.
// The useMetadata parameter is ignored.
func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error) {
if err := c.Init(ctx); err != nil {
return nil, err
}
@@ -270,11 +276,23 @@ func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID)
continue
}
// Always use accurate timestamp from headers since it's zero-cost
// NATS includes headers in LIST operations, so no extra API call needed
createdAt := objInfo.ModTime
if objInfo.Headers != nil {
if values, ok := objInfo.Headers[HeaderKeyTimestamp]; ok && len(values) > 0 {
if parsed, err := time.Parse(time.RFC3339Nano, values[0]); err == nil {
createdAt = parsed
}
}
}
fileInfos = append(fileInfos, &ltx.FileInfo{
Level: fileLevel,
MinTXID: minTXID,
MaxTXID: maxTXID,
Size: int64(objInfo.Size),
Level: fileLevel,
MinTXID: minTXID,
MaxTXID: maxTXID,
Size: int64(objInfo.Size),
CreatedAt: createdAt,
})
}
@@ -329,13 +347,27 @@ func (c *ReplicaClient) WriteLTXFile(ctx context.Context, level int, minTXID, ma
}
objectPath := c.ltxPath(level, minTXID, maxTXID)
startTime := time.Now()
// Wrap reader to count bytes
rc := internal.NewReadCounter(r)
// Use TeeReader to peek at LTX header while preserving data for upload
var buf bytes.Buffer
teeReader := io.TeeReader(r, &buf)
// Extract timestamp from LTX header
hdr, _, err := ltx.PeekHeader(teeReader)
if err != nil {
return nil, fmt.Errorf("extract timestamp from LTX header: %w", err)
}
timestamp := time.UnixMilli(hdr.Timestamp).UTC()
// Combine buffered data with rest of reader
rc := internal.NewReadCounter(io.MultiReader(&buf, r))
// Store timestamp in NATS object headers for accurate timestamp retrieval
objectInfo, err := c.objectStore.Put(ctx, jetstream.ObjectMeta{
Name: objectPath,
Headers: map[string][]string{
HeaderKeyTimestamp: {timestamp.Format(time.RFC3339Nano)},
},
}, rc)
if err != nil {
return nil, fmt.Errorf("failed to put object %s: %w", objectPath, err)
@@ -350,7 +382,7 @@ func (c *ReplicaClient) WriteLTXFile(ctx context.Context, level int, minTXID, ma
MinTXID: minTXID,
MaxTXID: maxTXID,
Size: int64(objectInfo.Size),
CreatedAt: startTime.UTC(),
CreatedAt: timestamp,
}, nil
}

15
replica.go
View File

@@ -191,7 +191,8 @@ func (r *Replica) calcPos(ctx context.Context) (pos ltx.Pos, err error) {
// MaxLTXFileInfo returns metadata about the last LTX file for a given level.
// Returns nil if no files exist for the level.
func (r *Replica) MaxLTXFileInfo(ctx context.Context, level int) (info ltx.FileInfo, err error) {
itr, err := r.Client.LTXFiles(ctx, level, 0)
// Normal operation - use fast timestamps
itr, err := r.Client.LTXFiles(ctx, level, 0, false)
if err != nil {
return info, err
}
@@ -340,7 +341,8 @@ func (r *Replica) monitor(ctx context.Context) {
func (r *Replica) CreatedAt(ctx context.Context) (time.Time, error) {
var min time.Time
itr, err := r.Client.LTXFiles(ctx, 0, 0)
// Normal operation - use fast timestamps
itr, err := r.Client.LTXFiles(ctx, 0, 0, false)
if err != nil {
return min, err
}
@@ -355,7 +357,8 @@ func (r *Replica) CreatedAt(ctx context.Context) (time.Time, error) {
// TimeBounds returns the creation time & last updated time.
// Returns zero time if LTX files exist.
func (r *Replica) TimeBounds(ctx context.Context) (createdAt, updatedAt time.Time, err error) {
itr, err := r.Client.LTXFiles(ctx, 0, 0)
// Normal operation - use fast timestamps
itr, err := r.Client.LTXFiles(ctx, 0, 0, false)
if err != nil {
return createdAt, updatedAt, err
}
@@ -491,7 +494,8 @@ func CalcRestorePlan(ctx context.Context, client ReplicaClient, txID ltx.TXID, t
logger = logger.With("target", txID)
// Start with latest snapshot before target TXID or timestamp.
if a, err := FindLTXFiles(ctx, client, SnapshotLevel, func(info *ltx.FileInfo) (bool, error) {
// Pass useMetadata flag to enable accurate timestamp fetching for timestamp-based restore.
if a, err := FindLTXFiles(ctx, client, SnapshotLevel, !timestamp.IsZero(), func(info *ltx.FileInfo) (bool, error) {
logger.Debug("finding snapshot before target TXID or timestamp", "snapshot", info.MaxTXID)
if txID != 0 {
return info.MaxTXID <= txID, nil
@@ -513,7 +517,8 @@ func CalcRestorePlan(ctx context.Context, client ReplicaClient, txID ltx.TXID, t
for level := maxLevel; level >= 0; level-- {
logger.Debug("finding ltx files for level", "level", level)
a, err := FindLTXFiles(ctx, client, level, func(info *ltx.FileInfo) (bool, error) {
// Pass useMetadata flag to enable accurate timestamp fetching for timestamp-based restore.
a, err := FindLTXFiles(ctx, client, level, !timestamp.IsZero(), func(info *ltx.FileInfo) (bool, error) {
if info.MaxTXID <= infos.MaxTXID() { // skip if already included in previous levels
return false, nil
}

11
replica_client.go
View File

@@ -21,7 +21,9 @@ type ReplicaClient interface {
// LTXFiles returns an iterator of all LTX files on the replica for a given level.
// If seek is specified, the iterator start from the given TXID or the next available if not found.
LTXFiles(ctx context.Context, level int, seek ltx.TXID) (ltx.FileIterator, error)
// If useMetadata is true, the iterator fetches accurate timestamps from metadata for timestamp-based restore.
// When false, the iterator uses fast timestamps (LastModified/Created/ModTime) for normal operations.
LTXFiles(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error)
// OpenLTXFile returns a reader that contains an LTX file at a given TXID.
// If seek is specified, the reader will start at the given offset.
@@ -40,8 +42,11 @@ type ReplicaClient interface {
}
// FindLTXFiles returns a list of files that match filter.
func FindLTXFiles(ctx context.Context, client ReplicaClient, level int, filter func(*ltx.FileInfo) (bool, error)) ([]*ltx.FileInfo, error) {
itr, err := client.LTXFiles(ctx, level, 0)
// The useMetadata parameter is passed through to LTXFiles to control whether accurate timestamps
// are fetched from metadata. When true (timestamp-based restore), accurate timestamps are required.
// When false (normal operations), fast timestamps are sufficient.
func FindLTXFiles(ctx context.Context, client ReplicaClient, level int, useMetadata bool, filter func(*ltx.FileInfo) (bool, error)) ([]*ltx.FileInfo, error) {
itr, err := client.LTXFiles(ctx, level, 0, useMetadata)
if err != nil {
return nil, err
}

157
replica_client_test.go
View File

@@ -17,26 +17,42 @@ import (
"github.com/benbjohnson/litestream/s3"
)
// createLTXData creates a minimal valid LTX file with a header for testing.
// The data parameter is appended after the header for testing purposes.
func createLTXData(minTXID, maxTXID ltx.TXID, data []byte) []byte {
hdr := ltx.Header{
Version: 1,
PageSize: 4096,
Commit: 1,
MinTXID: minTXID,
MaxTXID: maxTXID,
Timestamp: time.Now().UnixMilli(),
}
headerBytes, _ := hdr.MarshalBinary()
return append(headerBytes, data...)
}
func TestReplicaClient_LTX(t *testing.T) {
RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
t.Helper()
t.Parallel()
// Write files out of order to check for sorting.
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(4), ltx.TXID(8), strings.NewReader(`67`)); err != nil {
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(4), ltx.TXID(8), bytes.NewReader(createLTXData(4, 8, []byte(`67`)))); err != nil {
t.Fatal(err)
}
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(1), ltx.TXID(1), strings.NewReader(``)); err != nil {
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(1), ltx.TXID(1), bytes.NewReader(createLTXData(1, 1, []byte(``)))); err != nil {
t.Fatal(err)
}
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(9), ltx.TXID(9), strings.NewReader(`xyz`)); err != nil {
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(9), ltx.TXID(9), bytes.NewReader(createLTXData(9, 9, []byte(`xyz`)))); err != nil {
t.Fatal(err)
}
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(2), ltx.TXID(3), strings.NewReader(`12345`)); err != nil {
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(2), ltx.TXID(3), bytes.NewReader(createLTXData(2, 3, []byte(`12345`)))); err != nil {
t.Fatal(err)
}
itr, err := c.LTXFiles(context.Background(), 0, 0)
itr, err := c.LTXFiles(context.Background(), 0, 0, false)
if err != nil {
t.Fatal(err)
}
@@ -50,17 +66,30 @@ func TestReplicaClient_LTX(t *testing.T) {
t.Fatalf("len=%v, want %v", got, want)
}
if got, want := stripLTXFileInfo(a[0]), (&ltx.FileInfo{MinTXID: 1, MaxTXID: 1, Size: 0}); *got != *want {
t.Fatalf("Index=%v, want %v", got, want)
// Check that files are sorted by MinTXID (Size no longer checked since we add LTX headers)
if got, want := a[0].MinTXID, ltx.TXID(1); got != want {
t.Fatalf("Index[0].MinTXID=%v, want %v", got, want)
}
if got, want := stripLTXFileInfo(a[1]), (&ltx.FileInfo{MinTXID: 2, MaxTXID: 3, Size: 5}); *got != *want {
t.Fatalf("Index=%v, want %v", got, want)
if got, want := a[0].MaxTXID, ltx.TXID(1); got != want {
t.Fatalf("Index[0].MaxTXID=%v, want %v", got, want)
}
if got, want := stripLTXFileInfo(a[2]), (&ltx.FileInfo{MinTXID: 4, MaxTXID: 8, Size: 2}); *got != *want {
t.Fatalf("Index=%v, want %v", got, want)
if got, want := a[1].MinTXID, ltx.TXID(2); got != want {
t.Fatalf("Index[1].MinTXID=%v, want %v", got, want)
}
if got, want := stripLTXFileInfo(a[3]), (&ltx.FileInfo{MinTXID: 9, MaxTXID: 9, Size: 3}); *got != *want {
t.Fatalf("Index=%v, want %v", got, want)
if got, want := a[1].MaxTXID, ltx.TXID(3); got != want {
t.Fatalf("Index[1].MaxTXID=%v, want %v", got, want)
}
if got, want := a[2].MinTXID, ltx.TXID(4); got != want {
t.Fatalf("Index[2].MinTXID=%v, want %v", got, want)
}
if got, want := a[2].MaxTXID, ltx.TXID(8); got != want {
t.Fatalf("Index[2].MaxTXID=%v, want %v", got, want)
}
if got, want := a[3].MinTXID, ltx.TXID(9); got != want {
t.Fatalf("Index[3].MinTXID=%v, want %v", got, want)
}
if got, want := a[3].MaxTXID, ltx.TXID(9); got != want {
t.Fatalf("Index[3].MaxTXID=%v, want %v", got, want)
}
if err := itr.Close(); err != nil {
@@ -72,7 +101,7 @@ func TestReplicaClient_LTX(t *testing.T) {
t.Helper()
t.Parallel()
itr, err := c.LTXFiles(context.Background(), 0, 0)
itr, err := c.LTXFiles(context.Background(), 0, 0, false)
if err != nil {
t.Fatal(err)
}
@@ -89,7 +118,11 @@ func TestReplicaClient_WriteLTXFile(t *testing.T) {
t.Helper()
t.Parallel()
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(1), ltx.TXID(2), strings.NewReader(`foobar`)); err != nil {
testData := []byte(`foobar`)
ltxData := createLTXData(1, 2, testData)
expectedContent := ltxData
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(1), ltx.TXID(2), bytes.NewReader(expectedContent)); err != nil {
t.Fatal(err)
}
@@ -108,7 +141,7 @@ func TestReplicaClient_WriteLTXFile(t *testing.T) {
t.Fatal(err)
}
if got, want := string(buf), `foobar`; got != want {
if got, want := string(buf), string(expectedContent); got != want {
t.Fatalf("data=%q, want %q", got, want)
}
})
@@ -118,7 +151,12 @@ func TestReplicaClient_OpenLTXFile(t *testing.T) {
RunWithReplicaClient(t, "OK", func(t *testing.T, c litestream.ReplicaClient) {
t.Helper()
t.Parallel()
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(1), ltx.TXID(2), strings.NewReader(`foobar`)); err != nil {
testData := []byte(`foobar`)
ltxData := createLTXData(1, 2, testData)
expectedContent := ltxData
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(1), ltx.TXID(2), bytes.NewReader(expectedContent)); err != nil {
t.Fatal(err)
}
@@ -130,7 +168,7 @@ func TestReplicaClient_OpenLTXFile(t *testing.T) {
if buf, err := io.ReadAll(r); err != nil {
t.Fatal(err)
} else if got, want := string(buf), "foobar"; got != want {
} else if got, want := string(buf), string(expectedContent); got != want {
t.Fatalf("ReadAll=%v, want %v", got, want)
}
})
@@ -150,10 +188,10 @@ func TestReplicaClient_DeleteWALSegments(t *testing.T) {
t.Helper()
t.Parallel()
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(1), ltx.TXID(2), strings.NewReader(`foo`)); err != nil {
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(1), ltx.TXID(2), bytes.NewReader(createLTXData(1, 2, []byte(`foo`)))); err != nil {
t.Fatal(err)
}
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(3), ltx.TXID(4), strings.NewReader(`bar`)); err != nil {
if _, err := c.WriteLTXFile(context.Background(), 0, ltx.TXID(3), ltx.TXID(4), bytes.NewReader(createLTXData(3, 4, []byte(`bar`)))); err != nil {
t.Fatal(err)
}
@@ -191,10 +229,81 @@ func RunWithReplicaClient(t *testing.T, name string, fn func(*testing.T, litestr
})
}
func stripLTXFileInfo(info *ltx.FileInfo) *ltx.FileInfo {
other := *info
other.CreatedAt = time.Time{}
return &other
// TestReplicaClient_TimestampPreservation verifies that LTX file timestamps are preserved
// during write and read operations. This is critical for point-in-time restoration (#771).
func TestReplicaClient_TimestampPreservation(t *testing.T) {
RunWithReplicaClient(t, "PreservesTimestamp", func(t *testing.T, c litestream.ReplicaClient) {
t.Helper()
t.Parallel()
ctx := context.Background()
// Create an LTX file with a specific timestamp
// Use a timestamp from the past to ensure it's different from write time
expectedTimestamp := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond)
// Create a minimal LTX file with header containing the timestamp
hdr := ltx.Header{
Version: 1,
PageSize: 4096,
Commit: 1,
MinTXID: 1,
MaxTXID: 1,
Timestamp: expectedTimestamp.UnixMilli(),
}
// Marshal LTX header
headerBytes, err := hdr.MarshalBinary()
if err != nil {
t.Fatal(err)
}
// Combine header with minimal body to create valid LTX file
buf := bytes.NewReader(headerBytes)
// Write to storage backend
info, err := c.WriteLTXFile(ctx, 0, ltx.TXID(1), ltx.TXID(1), buf)
if err != nil {
t.Fatal(err)
}
// For File backend, timestamp should be preserved immediately
// For cloud backends (S3, GCS, Azure, NATS), timestamp is stored in metadata
// Verify the returned FileInfo has correct timestamp
if info.CreatedAt.IsZero() {
t.Fatal("WriteLTXFile returned zero timestamp")
}
// Read back via LTXFiles and verify timestamp is preserved
itr, err := c.LTXFiles(ctx, 0, 0, false)
if err != nil {
t.Fatal(err)
}
defer itr.Close()
var found *ltx.FileInfo
for itr.Next() {
item := itr.Item()
if item.MinTXID == 1 && item.MaxTXID == 1 {
found = item
break
}
}
if err := itr.Close(); err != nil {
t.Fatal(err)
}
if found == nil {
t.Fatal("LTX file not found in iteration")
}
// Verify timestamp was preserved (allow 1 second drift for precision)
timeDiff := found.CreatedAt.Sub(expectedTimestamp)
if timeDiff.Abs() > time.Second {
t.Errorf("Timestamp not preserved: expected %v, got %v (diff: %v)",
expectedTimestamp, found.CreatedAt, timeDiff)
}
})
}
// TestReplicaClient_S3_UploaderConfig tests S3 uploader configuration for large files

12
replica_test.go
View File

@@ -256,7 +256,7 @@ func TestReplica_CalcRestorePlan(t *testing.T) {
t.Run("SnapshotOnly", func(t *testing.T) {
var c mock.ReplicaClient
r := litestream.NewReplicaWithClient(db, &c)
c.LTXFilesFunc = func(ctx context.Context, level int, seek ltx.TXID) (ltx.FileIterator, error) {
c.LTXFilesFunc = func(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error) {
if level == litestream.SnapshotLevel {
return ltx.NewFileInfoSliceIterator([]*ltx.FileInfo{{
Level: litestream.SnapshotLevel,
@@ -284,7 +284,7 @@ func TestReplica_CalcRestorePlan(t *testing.T) {
t.Run("SnapshotAndIncremental", func(t *testing.T) {
var c mock.ReplicaClient
r := litestream.NewReplicaWithClient(db, &c)
c.LTXFilesFunc = func(ctx context.Context, level int, seek ltx.TXID) (ltx.FileIterator, error) {
c.LTXFilesFunc = func(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error) {
switch level {
case litestream.SnapshotLevel:
return ltx.NewFileInfoSliceIterator([]*ltx.FileInfo{
@@ -334,7 +334,7 @@ func TestReplica_CalcRestorePlan(t *testing.T) {
t.Run("ErrNonContiguousFiles", func(t *testing.T) {
var c mock.ReplicaClient
r := litestream.NewReplicaWithClient(db, &c)
c.LTXFilesFunc = func(ctx context.Context, level int, seek ltx.TXID) (ltx.FileIterator, error) {
c.LTXFilesFunc = func(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error) {
switch level {
case litestream.SnapshotLevel:
return ltx.NewFileInfoSliceIterator([]*ltx.FileInfo{
@@ -358,7 +358,7 @@ func TestReplica_CalcRestorePlan(t *testing.T) {
t.Run("ErrTxNotAvailable", func(t *testing.T) {
var c mock.ReplicaClient
r := litestream.NewReplicaWithClient(db, &c)
c.LTXFilesFunc = func(ctx context.Context, level int, seek ltx.TXID) (ltx.FileIterator, error) {
c.LTXFilesFunc = func(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error) {
switch level {
case litestream.SnapshotLevel:
return ltx.NewFileInfoSliceIterator([]*ltx.FileInfo{
@@ -377,7 +377,7 @@ func TestReplica_CalcRestorePlan(t *testing.T) {
t.Run("ErrNoFiles", func(t *testing.T) {
var c mock.ReplicaClient
c.LTXFilesFunc = func(ctx context.Context, level int, seek ltx.TXID) (ltx.FileIterator, error) {
c.LTXFilesFunc = func(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error) {
return ltx.NewFileInfoSliceIterator(nil), nil
}
r := litestream.NewReplicaWithClient(db, &c)
@@ -413,7 +413,7 @@ func TestReplica_ContextCancellationNoLogs(t *testing.T) {
// Create a replica with a mock client that simulates context cancellation during Sync
syncCount := 0
mockClient := &mock.ReplicaClient{
LTXFilesFunc: func(ctx context.Context, level int, seek ltx.TXID) (ltx.FileIterator, error) {
LTXFilesFunc: func(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error) {
syncCount++
// First few calls succeed, then return context.Canceled
if syncCount <= 2 {

93
s3/replica_client.go
View File

@@ -1,6 +1,7 @@
package s3
import (
"bytes"
"context"
"crypto/tls"
"errors"
@@ -34,6 +35,9 @@ import (
// ReplicaClientType is the client type for this package.
const ReplicaClientType = "s3"
// MetadataKeyTimestamp is the metadata key for storing LTX file timestamps in S3.
const MetadataKeyTimestamp = "litestream-timestamp"
// MaxKeys is the number of keys S3 can operate on per batch.
const MaxKeys = 1000
@@ -271,11 +275,13 @@ func (c *ReplicaClient) findBucketRegion(ctx context.Context, bucket string) (st
}
// LTXFiles returns an iterator over all LTX files on the replica for the given level.
func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID) (ltx.FileIterator, error) {
// When useMetadata is true, fetches accurate timestamps from S3 metadata via HeadObject.
// When false, uses fast LastModified timestamps from LIST operation.
func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error) {
if err := c.Init(ctx); err != nil {
return nil, err
}
return newFileIterator(ctx, c, level, seek), nil
return newFileIterator(ctx, c, level, seek, useMetadata), nil
}
// OpenLTXFile returns a reader for an LTX file
@@ -310,19 +316,39 @@ func (c *ReplicaClient) OpenLTXFile(ctx context.Context, level int, minTXID, max
}
// WriteLTXFile writes an LTX file to the replica.
// Extracts timestamp from LTX header and stores it in S3 metadata to preserve original creation time.
func (c *ReplicaClient) WriteLTXFile(ctx context.Context, level int, minTXID, maxTXID ltx.TXID, r io.Reader) (*ltx.FileInfo, error) {
if err := c.Init(ctx); err != nil {
return nil, err
}
rc := internal.NewReadCounter(r)
// Use TeeReader to peek at LTX header while preserving data for upload
var buf bytes.Buffer
teeReader := io.TeeReader(r, &buf)
// Extract timestamp from LTX header
hdr, _, err := ltx.PeekHeader(teeReader)
if err != nil {
return nil, fmt.Errorf("extract timestamp from LTX header: %w", err)
}
timestamp := time.UnixMilli(hdr.Timestamp).UTC()
// Combine buffered data with rest of reader
rc := internal.NewReadCounter(io.MultiReader(&buf, r))
filename := ltx.FormatFilename(minTXID, maxTXID)
key := c.Path + "/" + fmt.Sprintf("%04x/%s", level, filename)
// Store timestamp in S3 metadata for accurate timestamp retrieval
metadata := map[string]string{
MetadataKeyTimestamp: timestamp.Format(time.RFC3339Nano),
}
out, err := c.uploader.Upload(ctx, &s3.PutObjectInput{
Bucket: aws.String(c.Bucket),
Key: aws.String(key),
Body: rc,
Bucket: aws.String(c.Bucket),
Key: aws.String(key),
Body: rc,
Metadata: metadata,
})
if err != nil {
return nil, fmt.Errorf("s3: upload to %s: %w", key, err)
@@ -334,7 +360,7 @@ func (c *ReplicaClient) WriteLTXFile(ctx context.Context, level int, minTXID, ma
MinTXID: minTXID,
MaxTXID: maxTXID,
Size: rc.N(),
CreatedAt: time.Now(),
CreatedAt: timestamp,
}
internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
@@ -435,11 +461,12 @@ func (c *ReplicaClient) DeleteAll(ctx context.Context) error {
// fileIterator represents an iterator over LTX files in S3.
type fileIterator struct {
ctx context.Context
cancel context.CancelFunc
client *ReplicaClient
level int
seek ltx.TXID
ctx context.Context
cancel context.CancelFunc
client *ReplicaClient
level int
seek ltx.TXID
useMetadata bool // When true, fetch accurate timestamps from metadata
paginator *s3.ListObjectsV2Paginator
page *s3.ListObjectsV2Output
@@ -450,15 +477,16 @@ type fileIterator struct {
info *ltx.FileInfo
}
func newFileIterator(ctx context.Context, client *ReplicaClient, level int, seek ltx.TXID) *fileIterator {
func newFileIterator(ctx context.Context, client *ReplicaClient, level int, seek ltx.TXID, useMetadata bool) *fileIterator {
ctx, cancel := context.WithCancel(ctx)
itr := &fileIterator{
ctx: ctx,
cancel: cancel,
client: client,
level: level,
seek: seek,
ctx: ctx,
cancel: cancel,
client: client,
level: level,
seek: seek,
useMetadata: useMetadata,
}
// Create paginator for listing objects with level prefix
@@ -532,7 +560,34 @@ func (itr *fileIterator) Next() bool {
// Set file info
info.Size = aws.ToInt64(obj.Size)
info.CreatedAt = aws.ToTime(obj.LastModified)
// Use fast LastModified timestamp by default
createdAt := aws.ToTime(obj.LastModified).UTC()
// Only fetch accurate timestamp from metadata when requested (timestamp-based restore)
if itr.useMetadata {
head, err := itr.client.s3.HeadObject(itr.ctx, &s3.HeadObjectInput{
Bucket: aws.String(itr.client.Bucket),
Key: obj.Key,
})
if err != nil {
itr.err = fmt.Errorf("fetch object metadata: %w", err)
return false
}
if head.Metadata != nil {
if ts, ok := head.Metadata[MetadataKeyTimestamp]; ok {
if parsed, err := time.Parse(time.RFC3339Nano, ts); err == nil {
createdAt = parsed
} else {
itr.err = fmt.Errorf("parse timestamp from metadata: %w", err)
return false
}
}
}
}
info.CreatedAt = createdAt
itr.info = info
return true
}

31
sftp/replica_client.go
View File

@@ -1,6 +1,7 @@
package sftp
import (
"bytes"
"context"
"errors"
"fmt"
@@ -167,7 +168,9 @@ func (c *ReplicaClient) DeleteAll(ctx context.Context) (err error) {
}
// LTXFiles returns an iterator over all available LTX files for a level.
func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID) (_ ltx.FileIterator, err error) {
// SFTP uses file ModTime for timestamps, which is set via Chtimes() to preserve original timestamp.
// The useMetadata parameter is ignored since ModTime always contains the accurate timestamp.
func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID, _ bool) (_ ltx.FileIterator, err error) {
defer func() { c.resetOnConnError(err) }()
sftpClient, err := c.Init(ctx)
@@ -198,7 +201,7 @@ func (c *ReplicaClient) LTXFiles(ctx context.Context, level int, seek ltx.TXID)
MinTXID: minTXID,
MaxTXID: maxTXID,
Size: fi.Size(),
CreatedAt: fi.ModTime().UTC(),
CreatedAt: fi.ModTime().UTC(), // ModTime contains accurate timestamp from Chtimes()
})
}
@@ -215,7 +218,20 @@ func (c *ReplicaClient) WriteLTXFile(ctx context.Context, level int, minTXID, ma
}
filename := litestream.LTXFilePath(c.Path, level, minTXID, maxTXID)
startTime := time.Now()
// Use TeeReader to peek at LTX header while preserving data for upload
var buf bytes.Buffer
teeReader := io.TeeReader(rd, &buf)
// Extract timestamp from LTX header
hdr, _, err := ltx.PeekHeader(teeReader)
if err != nil {
return nil, fmt.Errorf("extract timestamp from LTX header: %w", err)
}
timestamp := time.UnixMilli(hdr.Timestamp).UTC()
// Combine buffered data with rest of reader
fullReader := io.MultiReader(&buf, rd)
if err := sftpClient.MkdirAll(path.Dir(filename)); err != nil {
return nil, fmt.Errorf("sftp: cannot make parent snapshot directory %q: %w", path.Dir(filename), err)
@@ -227,13 +243,18 @@ func (c *ReplicaClient) WriteLTXFile(ctx context.Context, level int, minTXID, ma
}
defer f.Close()
n, err := io.Copy(f, rd)
n, err := io.Copy(f, fullReader)
if err != nil {
return nil, err
} else if err := f.Close(); err != nil {
return nil, err
}
// Set file ModTime to preserve original timestamp
if err := sftpClient.Chtimes(filename, timestamp, timestamp); err != nil {
return nil, fmt.Errorf("sftp: cannot set file timestamps: %w", err)
}
internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "PUT").Inc()
internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "PUT").Add(float64(n))
@@ -242,7 +263,7 @@ func (c *ReplicaClient) WriteLTXFile(ctx context.Context, level int, minTXID, ma
MinTXID: minTXID,
MaxTXID: maxTXID,
Size: n,
CreatedAt: startTime.UTC(),
CreatedAt: timestamp,
}, nil
}

2
store_compaction_remote_test.go
View File

@@ -130,7 +130,7 @@ func (c *delayedReplicaClient) key(level int, min, max ltx.TXID) string {
return fmt.Sprintf("%d:%s:%s", level, min.String(), max.String())
}
func (c *delayedReplicaClient) LTXFiles(_ context.Context, level int, seek ltx.TXID) (ltx.FileIterator, error) {
func (c *delayedReplicaClient) LTXFiles(_ context.Context, level int, seek ltx.TXID, _ bool) (ltx.FileIterator, error) {
c.mu.Lock()
defer c.mu.Unlock()

2
vfs.go
View File

@@ -278,7 +278,7 @@ func (f *VFSFile) pollReplicaClient(ctx context.Context) error {
f.logger.Debug("polling replica client", "txid", pos.TXID.String())
// Start reading from the next LTX file after the current position.
itr, err := f.client.LTXFiles(ctx, 0, f.pos.TXID+1)
itr, err := f.client.LTXFiles(ctx, 0, f.pos.TXID+1, false)
if err != nil {
return fmt.Errorf("ltx files: %w", err)
}