feat(vfs): add VFS compaction support via shared Compactor type (#979)

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Cory LaNou
2026-01-09 16:07:50 -06:00
committed by GitHub
parent e45a2922b9
commit 91f5e37843
7 changed files with 1242 additions and 123 deletions

View File

@@ -279,9 +279,9 @@ func DefaultConfig() Config {
defaultShutdownSyncInterval := litestream.DefaultShutdownSyncInterval
return Config{
Levels: []*CompactionLevelConfig{
{Interval: 30 * time.Second},
{Interval: 5 * time.Minute},
{Interval: 1 * time.Hour},
{Interval: litestream.DefaultCompactionLevels[1].Interval},
{Interval: litestream.DefaultCompactionLevels[2].Interval},
{Interval: litestream.DefaultCompactionLevels[3].Interval},
},
Snapshot: SnapshotConfig{
Interval: &defaultSnapshotInterval,

View File

@@ -8,6 +8,16 @@ import (
// SnapshotLevel represents the level which full snapshots are held.
const SnapshotLevel = 9
// DefaultCompactionLevels provides the canonical default compaction configuration.
// Level 0 is raw LTX files, higher levels compact at increasing intervals.
// These values are also used by cmd/litestream DefaultConfig().
var DefaultCompactionLevels = CompactionLevels{
{Level: 0, Interval: 0},
{Level: 1, Interval: 30 * time.Second},
{Level: 2, Interval: 5 * time.Minute},
{Level: 3, Interval: time.Hour},
}
// CompactionLevel represents a single part of a multi-level compaction.
// Each level merges LTX files from the previous level into larger time granularities.
type CompactionLevel struct {

355
compactor.go Normal file
View File

@@ -0,0 +1,355 @@
package litestream
import (
"context"
"fmt"
"io"
"log/slog"
"os"
"time"
"github.com/superfly/ltx"
)
// Compactor handles compaction and retention for LTX files.
// It operates solely through the ReplicaClient interface, making it
// suitable for both DB (with local file caching) and VFS (remote-only).
type Compactor struct {
client ReplicaClient
logger *slog.Logger
// LocalFileOpener optionally opens a local LTX file for compaction.
// If nil or returns os.ErrNotExist, falls back to remote.
// This is used by DB to prefer local files over remote for consistency.
LocalFileOpener func(level int, minTXID, maxTXID ltx.TXID) (io.ReadCloser, error)
// LocalFileDeleter optionally deletes local LTX files after retention.
// If nil, only remote files are deleted.
LocalFileDeleter func(level int, minTXID, maxTXID ltx.TXID) error
// CacheGetter optionally retrieves cached MaxLTXFileInfo for a level.
// If nil, max file info is always fetched from remote.
CacheGetter func(level int) (*ltx.FileInfo, bool)
// CacheSetter optionally stores MaxLTXFileInfo for a level.
// If nil, max file info is not cached.
CacheSetter func(level int, info *ltx.FileInfo)
}
// NewCompactor creates a new Compactor with the given client and logger.
func NewCompactor(client ReplicaClient, logger *slog.Logger) *Compactor {
if logger == nil {
logger = slog.Default()
}
return &Compactor{
client: client,
logger: logger,
}
}
// Client returns the underlying ReplicaClient.
func (c *Compactor) Client() ReplicaClient {
return c.client
}
// SetClient updates the ReplicaClient.
// This is used by DB when the Replica is assigned after construction.
func (c *Compactor) SetClient(client ReplicaClient) {
c.client = client
}
// MaxLTXFileInfo returns metadata for the last LTX file in a level.
// Uses cache if available, otherwise fetches from remote.
func (c *Compactor) MaxLTXFileInfo(ctx context.Context, level int) (ltx.FileInfo, error) {
if c.CacheGetter != nil {
if info, ok := c.CacheGetter(level); ok {
return *info, nil
}
}
itr, err := c.client.LTXFiles(ctx, level, 0, false)
if err != nil {
return ltx.FileInfo{}, err
}
defer itr.Close()
var info ltx.FileInfo
for itr.Next() {
item := itr.Item()
if item.MaxTXID > info.MaxTXID {
info = *item
}
}
if c.CacheSetter != nil && info.MaxTXID > 0 {
c.CacheSetter(level, &info)
}
return info, itr.Close()
}
// Compact compacts source level files into the destination level.
// Returns ErrNoCompaction if there are no files to compact.
func (c *Compactor) Compact(ctx context.Context, dstLevel int) (*ltx.FileInfo, error) {
srcLevel := dstLevel - 1
prevMaxInfo, err := c.MaxLTXFileInfo(ctx, dstLevel)
if err != nil {
return nil, fmt.Errorf("cannot determine max ltx file for destination level: %w", err)
}
seekTXID := prevMaxInfo.MaxTXID + 1
itr, err := c.client.LTXFiles(ctx, srcLevel, seekTXID, false)
if err != nil {
return nil, fmt.Errorf("source ltx files after %s: %w", seekTXID, err)
}
defer itr.Close()
var rdrs []io.Reader
defer func() {
for _, rd := range rdrs {
if closer, ok := rd.(io.Closer); ok {
_ = closer.Close()
}
}
}()
var minTXID, maxTXID ltx.TXID
for itr.Next() {
info := itr.Item()
if minTXID == 0 || info.MinTXID < minTXID {
minTXID = info.MinTXID
}
if maxTXID == 0 || info.MaxTXID > maxTXID {
maxTXID = info.MaxTXID
}
if c.LocalFileOpener != nil {
if f, err := c.LocalFileOpener(srcLevel, info.MinTXID, info.MaxTXID); err == nil {
rdrs = append(rdrs, f)
continue
} else if !os.IsNotExist(err) {
return nil, fmt.Errorf("open local ltx file: %w", err)
}
}
f, err := c.client.OpenLTXFile(ctx, info.Level, info.MinTXID, info.MaxTXID, 0, 0)
if err != nil {
return nil, fmt.Errorf("open ltx file: %w", err)
}
rdrs = append(rdrs, f)
}
if len(rdrs) == 0 {
return nil, ErrNoCompaction
}
pr, pw := io.Pipe()
go func() {
comp, err := ltx.NewCompactor(pw, rdrs)
if err != nil {
pw.CloseWithError(fmt.Errorf("new ltx compactor: %w", err))
return
}
comp.HeaderFlags = ltx.HeaderFlagNoChecksum
_ = pw.CloseWithError(comp.Compact(ctx))
}()
info, err := c.client.WriteLTXFile(ctx, dstLevel, minTXID, maxTXID, pr)
if err != nil {
return nil, fmt.Errorf("write ltx file: %w", err)
}
if c.CacheSetter != nil {
c.CacheSetter(dstLevel, info)
}
return info, nil
}
// EnforceSnapshotRetention enforces retention of snapshot level files by timestamp.
// Files older than the retention duration are deleted (except the newest is always kept).
// Returns the minimum snapshot TXID still retained (useful for cascading retention to lower levels).
func (c *Compactor) EnforceSnapshotRetention(ctx context.Context, retention time.Duration) (ltx.TXID, error) {
timestamp := time.Now().Add(-retention)
c.logger.Debug("enforcing snapshot retention", "timestamp", timestamp)
itr, err := c.client.LTXFiles(ctx, SnapshotLevel, 0, false)
if err != nil {
return 0, fmt.Errorf("fetch ltx files: %w", err)
}
defer itr.Close()
var deleted []*ltx.FileInfo
var lastInfo *ltx.FileInfo
var minSnapshotTXID ltx.TXID
for itr.Next() {
info := itr.Item()
lastInfo = info
if info.CreatedAt.Before(timestamp) {
deleted = append(deleted, info)
continue
}
if minSnapshotTXID == 0 || info.MaxTXID < minSnapshotTXID {
minSnapshotTXID = info.MaxTXID
}
}
if len(deleted) > 0 && deleted[len(deleted)-1] == lastInfo {
deleted = deleted[:len(deleted)-1]
}
if err := c.client.DeleteLTXFiles(ctx, deleted); err != nil {
return 0, fmt.Errorf("remove ltx files: %w", err)
}
if c.LocalFileDeleter != nil {
for _, info := range deleted {
c.logger.Debug("deleting local ltx file",
"level", SnapshotLevel,
"minTXID", info.MinTXID,
"maxTXID", info.MaxTXID)
if err := c.LocalFileDeleter(SnapshotLevel, info.MinTXID, info.MaxTXID); err != nil {
c.logger.Error("failed to remove local ltx file", "error", err)
}
}
}
return minSnapshotTXID, nil
}
// EnforceRetentionByTXID deletes files at the given level with maxTXID below the target.
// Always keeps at least one file.
func (c *Compactor) EnforceRetentionByTXID(ctx context.Context, level int, txID ltx.TXID) error {
c.logger.Debug("enforcing retention", "level", level, "txid", txID)
itr, err := c.client.LTXFiles(ctx, level, 0, false)
if err != nil {
return fmt.Errorf("fetch ltx files: %w", err)
}
defer itr.Close()
var deleted []*ltx.FileInfo
var lastInfo *ltx.FileInfo
for itr.Next() {
info := itr.Item()
lastInfo = info
if info.MaxTXID < txID {
deleted = append(deleted, info)
continue
}
}
if len(deleted) > 0 && deleted[len(deleted)-1] == lastInfo {
deleted = deleted[:len(deleted)-1]
}
if err := c.client.DeleteLTXFiles(ctx, deleted); err != nil {
return fmt.Errorf("remove ltx files: %w", err)
}
if c.LocalFileDeleter != nil {
for _, info := range deleted {
c.logger.Debug("deleting local ltx file",
"level", level,
"minTXID", info.MinTXID,
"maxTXID", info.MaxTXID)
if err := c.LocalFileDeleter(level, info.MinTXID, info.MaxTXID); err != nil {
c.logger.Error("failed to remove local ltx file", "error", err)
}
}
}
return nil
}
// EnforceL0Retention retains L0 files based on L1 compaction progress and time.
// Files are only deleted if they have been compacted into L1 AND are older than retention.
// This ensures contiguous L0 coverage for VFS reads.
func (c *Compactor) EnforceL0Retention(ctx context.Context, retention time.Duration) error {
if retention <= 0 {
return nil
}
c.logger.Debug("enforcing l0 retention", "retention", retention)
itr, err := c.client.LTXFiles(ctx, 1, 0, false)
if err != nil {
return fmt.Errorf("fetch l1 files: %w", err)
}
var maxL1TXID ltx.TXID
for itr.Next() {
info := itr.Item()
if info.MaxTXID > maxL1TXID {
maxL1TXID = info.MaxTXID
}
}
if err := itr.Close(); err != nil {
return fmt.Errorf("close l1 iterator: %w", err)
}
if maxL1TXID == 0 {
return nil
}
threshold := time.Now().Add(-retention)
itr, err = c.client.LTXFiles(ctx, 0, 0, false)
if err != nil {
return fmt.Errorf("fetch l0 files: %w", err)
}
defer itr.Close()
var (
deleted []*ltx.FileInfo
lastInfo *ltx.FileInfo
processedAll = true
)
for itr.Next() {
info := itr.Item()
lastInfo = info
createdAt := info.CreatedAt
if createdAt.IsZero() {
createdAt = threshold
}
if createdAt.After(threshold) {
processedAll = false
break
}
if info.MaxTXID <= maxL1TXID {
deleted = append(deleted, info)
}
}
if processedAll && len(deleted) > 0 && lastInfo != nil && deleted[len(deleted)-1] == lastInfo {
deleted = deleted[:len(deleted)-1]
}
if len(deleted) == 0 {
return nil
}
if err := c.client.DeleteLTXFiles(ctx, deleted); err != nil {
return fmt.Errorf("remove expired l0 files: %w", err)
}
if c.LocalFileDeleter != nil {
for _, info := range deleted {
c.logger.Debug("deleting expired local l0 file",
"minTXID", info.MinTXID,
"maxTXID", info.MaxTXID)
if err := c.LocalFileDeleter(0, info.MinTXID, info.MaxTXID); err != nil {
c.logger.Error("failed to remove local l0 file", "error", err)
}
}
}
c.logger.Info("l0 retention enforced", "deleted_count", len(deleted), "max_l1_txid", maxL1TXID)
return nil
}

364
compactor_test.go Normal file
View File

@@ -0,0 +1,364 @@
package litestream_test
import (
"bytes"
"context"
"io"
"log/slog"
"testing"
"time"
"github.com/superfly/ltx"
"github.com/benbjohnson/litestream"
"github.com/benbjohnson/litestream/file"
)
func TestCompactor_Compact(t *testing.T) {
t.Run("L0ToL1", func(t *testing.T) {
client := file.NewReplicaClient(t.TempDir())
compactor := litestream.NewCompactor(client, slog.Default())
// Create test L0 files
createTestLTXFile(t, client, 0, 1, 1)
createTestLTXFile(t, client, 0, 2, 2)
info, err := compactor.Compact(context.Background(), 1)
if err != nil {
t.Fatal(err)
}
if info.Level != 1 {
t.Errorf("Level=%d, want 1", info.Level)
}
if info.MinTXID != 1 || info.MaxTXID != 2 {
t.Errorf("TXID range=%d-%d, want 1-2", info.MinTXID, info.MaxTXID)
}
})
t.Run("NoFiles", func(t *testing.T) {
client := file.NewReplicaClient(t.TempDir())
compactor := litestream.NewCompactor(client, slog.Default())
_, err := compactor.Compact(context.Background(), 1)
if err != litestream.ErrNoCompaction {
t.Errorf("err=%v, want ErrNoCompaction", err)
}
})
t.Run("L1ToL2", func(t *testing.T) {
client := file.NewReplicaClient(t.TempDir())
compactor := litestream.NewCompactor(client, slog.Default())
// Create L0 files
createTestLTXFile(t, client, 0, 1, 1)
createTestLTXFile(t, client, 0, 2, 2)
// Compact to L1
_, err := compactor.Compact(context.Background(), 1)
if err != nil {
t.Fatal(err)
}
// Create more L0 files
createTestLTXFile(t, client, 0, 3, 3)
// Compact to L1 again (should only include TXID 3)
info, err := compactor.Compact(context.Background(), 1)
if err != nil {
t.Fatal(err)
}
if info.MinTXID != 3 || info.MaxTXID != 3 {
t.Errorf("TXID range=%d-%d, want 3-3", info.MinTXID, info.MaxTXID)
}
// Now compact L1 to L2 (should include all from 1-3)
info, err = compactor.Compact(context.Background(), 2)
if err != nil {
t.Fatal(err)
}
if info.Level != 2 {
t.Errorf("Level=%d, want 2", info.Level)
}
if info.MinTXID != 1 || info.MaxTXID != 3 {
t.Errorf("TXID range=%d-%d, want 1-3", info.MinTXID, info.MaxTXID)
}
})
}
func TestCompactor_MaxLTXFileInfo(t *testing.T) {
t.Run("WithFiles", func(t *testing.T) {
client := file.NewReplicaClient(t.TempDir())
compactor := litestream.NewCompactor(client, slog.Default())
createTestLTXFile(t, client, 0, 1, 1)
createTestLTXFile(t, client, 0, 2, 2)
createTestLTXFile(t, client, 0, 3, 5)
info, err := compactor.MaxLTXFileInfo(context.Background(), 0)
if err != nil {
t.Fatal(err)
}
if info.MaxTXID != 5 {
t.Errorf("MaxTXID=%d, want 5", info.MaxTXID)
}
})
t.Run("NoFiles", func(t *testing.T) {
client := file.NewReplicaClient(t.TempDir())
compactor := litestream.NewCompactor(client, slog.Default())
info, err := compactor.MaxLTXFileInfo(context.Background(), 0)
if err != nil {
t.Fatal(err)
}
if info.MaxTXID != 0 {
t.Errorf("MaxTXID=%d, want 0", info.MaxTXID)
}
})
t.Run("WithCache", func(t *testing.T) {
client := file.NewReplicaClient(t.TempDir())
compactor := litestream.NewCompactor(client, slog.Default())
// Use callbacks for caching
cache := make(map[int]*ltx.FileInfo)
compactor.CacheGetter = func(level int) (*ltx.FileInfo, bool) {
info, ok := cache[level]
return info, ok
}
compactor.CacheSetter = func(level int, info *ltx.FileInfo) {
cache[level] = info
}
createTestLTXFile(t, client, 0, 1, 3)
// First call should populate cache
info, err := compactor.MaxLTXFileInfo(context.Background(), 0)
if err != nil {
t.Fatal(err)
}
if info.MaxTXID != 3 {
t.Errorf("MaxTXID=%d, want 3", info.MaxTXID)
}
// Second call should use cache
info, err = compactor.MaxLTXFileInfo(context.Background(), 0)
if err != nil {
t.Fatal(err)
}
if info.MaxTXID != 3 {
t.Errorf("MaxTXID=%d, want 3 (from cache)", info.MaxTXID)
}
})
}
func TestCompactor_EnforceRetentionByTXID(t *testing.T) {
t.Run("DeletesOldFiles", func(t *testing.T) {
client := file.NewReplicaClient(t.TempDir())
compactor := litestream.NewCompactor(client, slog.Default())
// Create files at L1
createTestLTXFile(t, client, 1, 1, 2)
createTestLTXFile(t, client, 1, 3, 5)
createTestLTXFile(t, client, 1, 6, 10)
// Enforce retention - delete files below TXID 5
err := compactor.EnforceRetentionByTXID(context.Background(), 1, 5)
if err != nil {
t.Fatal(err)
}
// Verify only the first file was deleted
info, err := compactor.MaxLTXFileInfo(context.Background(), 1)
if err != nil {
t.Fatal(err)
}
if info.MaxTXID != 10 {
t.Errorf("MaxTXID=%d, want 10", info.MaxTXID)
}
// Check that files starting from TXID 3 are still present
itr, err := client.LTXFiles(context.Background(), 1, 0, false)
if err != nil {
t.Fatal(err)
}
defer itr.Close()
var count int
for itr.Next() {
count++
}
if count != 2 {
t.Errorf("file count=%d, want 2", count)
}
})
t.Run("KeepsLastFile", func(t *testing.T) {
client := file.NewReplicaClient(t.TempDir())
compactor := litestream.NewCompactor(client, slog.Default())
// Create single file
createTestLTXFile(t, client, 1, 1, 2)
// Try to delete it - should keep at least one
err := compactor.EnforceRetentionByTXID(context.Background(), 1, 100)
if err != nil {
t.Fatal(err)
}
// Verify file still exists
info, err := compactor.MaxLTXFileInfo(context.Background(), 1)
if err != nil {
t.Fatal(err)
}
if info.MaxTXID != 2 {
t.Errorf("MaxTXID=%d, want 2 (last file should be kept)", info.MaxTXID)
}
})
}
func TestCompactor_EnforceL0Retention(t *testing.T) {
t.Run("DeletesCompactedFiles", func(t *testing.T) {
client := file.NewReplicaClient(t.TempDir())
compactor := litestream.NewCompactor(client, slog.Default())
// Create L0 files
createTestLTXFile(t, client, 0, 1, 1)
createTestLTXFile(t, client, 0, 2, 2)
createTestLTXFile(t, client, 0, 3, 3)
// Compact to L1
_, err := compactor.Compact(context.Background(), 1)
if err != nil {
t.Fatal(err)
}
// Enforce L0 retention with 0 duration (delete immediately)
err = compactor.EnforceL0Retention(context.Background(), 0)
if err != nil {
t.Fatal(err)
}
// L0 files compacted into L1 should be deleted (except last)
itr, err := client.LTXFiles(context.Background(), 0, 0, false)
if err != nil {
t.Fatal(err)
}
defer itr.Close()
var count int
for itr.Next() {
count++
}
// At least one file should remain
if count < 1 {
t.Errorf("file count=%d, want at least 1", count)
}
})
t.Run("SkipsIfNoL1", func(t *testing.T) {
client := file.NewReplicaClient(t.TempDir())
compactor := litestream.NewCompactor(client, slog.Default())
// Create L0 files without compacting to L1
createTestLTXFile(t, client, 0, 1, 1)
createTestLTXFile(t, client, 0, 2, 2)
// Enforce L0 retention - should do nothing since no L1 exists
err := compactor.EnforceL0Retention(context.Background(), 0)
if err != nil {
t.Fatal(err)
}
// All L0 files should still exist
itr, err := client.LTXFiles(context.Background(), 0, 0, false)
if err != nil {
t.Fatal(err)
}
defer itr.Close()
var count int
for itr.Next() {
count++
}
if count != 2 {
t.Errorf("file count=%d, want 2", count)
}
})
}
func TestCompactor_EnforceSnapshotRetention(t *testing.T) {
t.Run("DeletesOldSnapshots", func(t *testing.T) {
client := file.NewReplicaClient(t.TempDir())
compactor := litestream.NewCompactor(client, slog.Default())
// Create snapshot files with different timestamps
createTestLTXFileWithTimestamp(t, client, litestream.SnapshotLevel, 1, 5, time.Now().Add(-2*time.Hour))
createTestLTXFileWithTimestamp(t, client, litestream.SnapshotLevel, 1, 10, time.Now().Add(-30*time.Minute))
createTestLTXFileWithTimestamp(t, client, litestream.SnapshotLevel, 1, 15, time.Now().Add(-5*time.Minute))
// Enforce retention - keep snapshots from last hour
_, err := compactor.EnforceSnapshotRetention(context.Background(), time.Hour)
if err != nil {
t.Fatal(err)
}
// Count remaining snapshots
itr, err := client.LTXFiles(context.Background(), litestream.SnapshotLevel, 0, false)
if err != nil {
t.Fatal(err)
}
defer itr.Close()
var count int
for itr.Next() {
count++
}
// Should have 2 snapshots (the 30min and 5min old ones)
if count != 2 {
t.Errorf("snapshot count=%d, want 2", count)
}
})
}
// createTestLTXFile creates a minimal LTX file for testing.
func createTestLTXFile(t testing.TB, client litestream.ReplicaClient, level int, minTXID, maxTXID ltx.TXID) {
t.Helper()
createTestLTXFileWithTimestamp(t, client, level, minTXID, maxTXID, time.Now())
}
// createTestLTXFileWithTimestamp creates a minimal LTX file with a specific timestamp.
func createTestLTXFileWithTimestamp(t testing.TB, client litestream.ReplicaClient, level int, minTXID, maxTXID ltx.TXID, ts time.Time) {
t.Helper()
var buf bytes.Buffer
enc, err := ltx.NewEncoder(&buf)
if err != nil {
t.Fatal(err)
}
if err := enc.EncodeHeader(ltx.Header{
Version: ltx.Version,
Flags: ltx.HeaderFlagNoChecksum,
PageSize: 4096,
Commit: 1,
MinTXID: minTXID,
MaxTXID: maxTXID,
Timestamp: ts.UnixMilli(),
}); err != nil {
t.Fatal(err)
}
// Write a dummy page
if err := enc.EncodePage(ltx.PageHeader{Pgno: 1}, make([]byte, 4096)); err != nil {
t.Fatal(err)
}
if err := enc.Close(); err != nil {
t.Fatal(err)
}
if _, err := client.WriteLTXFile(context.Background(), level, minTXID, maxTXID, io.NopCloser(&buf)); err != nil {
t.Fatal(err)
}
}

168
db.go
View File

@@ -151,6 +151,10 @@ type DB struct {
// Must be set before calling Open().
Replica *Replica
// Compactor handles shared compaction logic.
// Created in NewDB, client set when Replica is assigned.
compactor *Compactor
// Shutdown sync retry settings.
// ShutdownSyncTimeout is the total time to retry syncing on shutdown.
// ShutdownSyncInterval is the time between retry attempts.
@@ -200,6 +204,22 @@ func NewDB(path string) *DB {
db.ctx, db.cancel = context.WithCancel(context.Background())
// Initialize compactor with nil client (will be set when Replica is assigned).
db.compactor = NewCompactor(nil, db.Logger)
db.compactor.LocalFileOpener = db.openLocalLTXFile
db.compactor.LocalFileDeleter = db.deleteLocalLTXFile
db.compactor.CacheGetter = func(level int) (*ltx.FileInfo, bool) {
db.maxLTXFileInfos.Lock()
defer db.maxLTXFileInfos.Unlock()
info, ok := db.maxLTXFileInfos.m[level]
return info, ok
}
db.compactor.CacheSetter = func(level int, info *ltx.FileInfo) {
db.maxLTXFileInfos.Lock()
defer db.maxLTXFileInfos.Unlock()
db.maxLTXFileInfos.m[level] = info
}
return db
}
@@ -246,6 +266,29 @@ func (db *DB) LTXPath(level int, minTXID, maxTXID ltx.TXID) string {
return filepath.Join(db.LTXLevelDir(level), ltx.FormatFilename(minTXID, maxTXID))
}
// openLocalLTXFile opens a local LTX file for reading.
// Used by the Compactor to prefer local files over remote.
func (db *DB) openLocalLTXFile(level int, minTXID, maxTXID ltx.TXID) (io.ReadCloser, error) {
return os.Open(db.LTXPath(level, minTXID, maxTXID))
}
// deleteLocalLTXFile deletes a local LTX file.
// Used by the Compactor for retention enforcement.
func (db *DB) deleteLocalLTXFile(level int, minTXID, maxTXID ltx.TXID) error {
path := db.LTXPath(level, minTXID, maxTXID)
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
// ensureCompactorClient ensures the compactor has the current replica client.
func (db *DB) ensureCompactorClient() {
if db.Replica != nil && db.compactor.Client() != db.Replica.Client {
db.compactor.SetClient(db.Replica.Client)
}
}
// MaxLTX returns the last LTX file written to level 0.
func (db *DB) MaxLTX() (minTXID, maxTXID ltx.TXID, err error) {
ents, err := os.ReadDir(db.LTXLevelDir(0))
@@ -1696,87 +1739,12 @@ func (db *DB) SnapshotReader(ctx context.Context) (ltx.Pos, io.Reader, error) {
// Returns metadata for the newly written compaction file. Returns ErrNoCompaction
// if no new files are available to be compacted.
func (db *DB) Compact(ctx context.Context, dstLevel int) (*ltx.FileInfo, error) {
srcLevel := dstLevel - 1
db.ensureCompactorClient()
prevMaxInfo, err := db.Replica.MaxLTXFileInfo(ctx, dstLevel)
info, err := db.compactor.Compact(ctx, dstLevel)
if err != nil {
return nil, fmt.Errorf("cannot determine max ltx file for destination level: %w", err)
return nil, err
}
seekTXID := prevMaxInfo.MaxTXID + 1
// Collect files after last compaction.
// Normal operation - use fast timestamps
itr, err := db.Replica.Client.LTXFiles(ctx, srcLevel, seekTXID, false)
if err != nil {
return nil, fmt.Errorf("source ltx files after %s: %w", seekTXID, err)
}
defer itr.Close()
// Ensure all readers are closed by the end, even if an error occurs.
var rdrs []io.Reader
defer func() {
for _, rd := range rdrs {
if closer, ok := rd.(io.Closer); ok {
_ = closer.Close()
}
}
}()
// Build a list of input files to compact from.
var minTXID, maxTXID ltx.TXID
for itr.Next() {
info := itr.Item()
// Track TXID bounds of all files being compacted.
if minTXID == 0 || info.MinTXID < minTXID {
minTXID = info.MinTXID
}
if maxTXID == 0 || info.MaxTXID > maxTXID {
maxTXID = info.MaxTXID
}
// Prefer the on-disk LTX file when available so compaction is not subject
// to eventual consistency quirks of remote storage providers. Fallback to
// downloading the file from the replica client if the local copy has been
// removed.
if f, err := os.Open(db.LTXPath(srcLevel, info.MinTXID, info.MaxTXID)); err == nil {
rdrs = append(rdrs, f)
continue
} else if !os.IsNotExist(err) {
return nil, fmt.Errorf("open local ltx file: %w", err)
}
f, err := db.Replica.Client.OpenLTXFile(ctx, info.Level, info.MinTXID, info.MaxTXID, 0, 0)
if err != nil {
return nil, fmt.Errorf("open ltx file: %w", err)
}
rdrs = append(rdrs, f)
}
if len(rdrs) == 0 {
return nil, ErrNoCompaction
}
// Stream compaction to destination in level.
pr, pw := io.Pipe()
go func() {
c, err := ltx.NewCompactor(pw, rdrs)
if err != nil {
pw.CloseWithError(fmt.Errorf("new ltx compactor: %w", err))
return
}
c.HeaderFlags = ltx.HeaderFlagNoChecksum
_ = pw.CloseWithError(c.Compact(ctx))
}()
info, err := db.Replica.Client.WriteLTXFile(ctx, dstLevel, minTXID, maxTXID, pr)
if err != nil {
return nil, fmt.Errorf("write ltx file: %w", err)
}
// Cache last metadata for the level.
db.maxLTXFileInfos.Lock()
db.maxLTXFileInfos.m[dstLevel] = info
db.maxLTXFileInfos.Unlock()
// If this is L1, clean up L0 files using the time-based retention policy.
if dstLevel == 1 {
@@ -1955,48 +1923,8 @@ func (db *DB) EnforceL0RetentionByTime(ctx context.Context) error {
// EnforceRetentionByTXID enforces retention so that any LTX files below
// the target TXID are deleted. Always keep at least one file.
func (db *DB) EnforceRetentionByTXID(ctx context.Context, level int, txID ltx.TXID) (err error) {
db.Logger.Debug("enforcing retention", "level", level, "txid", txID)
// Normal operation - use fast timestamps
itr, err := db.Replica.Client.LTXFiles(ctx, level, 0, false)
if err != nil {
return fmt.Errorf("fetch ltx files: %w", err)
}
defer itr.Close()
var deleted []*ltx.FileInfo
var lastInfo *ltx.FileInfo
for itr.Next() {
info := itr.Item()
lastInfo = info
// If this file's maxTXID is below the target TXID, mark it for deletion.
if info.MaxTXID < txID {
deleted = append(deleted, info)
continue
}
}
// Ensure we don't delete the last file.
if len(deleted) > 0 && deleted[len(deleted)-1] == lastInfo {
deleted = deleted[:len(deleted)-1]
}
// Remove all files marked for deletion from both remote and local storage.
if err := db.Replica.Client.DeleteLTXFiles(ctx, deleted); err != nil {
return fmt.Errorf("remove ltx files: %w", err)
}
for _, info := range deleted {
localPath := db.LTXPath(level, info.MinTXID, info.MaxTXID)
db.Logger.Debug("deleting local ltx file", "level", level, "minTXID", info.MinTXID, "maxTXID", info.MaxTXID, "path", localPath)
if err := os.Remove(localPath); err != nil && !os.IsNotExist(err) {
db.Logger.Error("failed to remove local ltx file", "path", localPath, "error", err)
}
}
return nil
db.ensureCompactorClient()
return db.compactor.EnforceRetentionByTXID(ctx, level, txID)
}
// monitor runs in a separate goroutine and monitors the database & WAL.

270
vfs.go
View File

@@ -83,6 +83,26 @@ type VFS struct {
// If empty and HydrationEnabled is true, a temp file will be used.
HydrationPath string
// CompactionEnabled activates background compaction for the VFS.
// Requires WriteEnabled to be true.
CompactionEnabled bool
// CompactionLevels defines the compaction intervals for each level.
// If nil, uses default compaction levels.
CompactionLevels CompactionLevels
// SnapshotInterval is how often to create full database snapshots.
// Set to 0 to disable automatic snapshots.
SnapshotInterval time.Duration
// SnapshotRetention is how long to keep old snapshots.
// Set to 0 to keep all snapshots.
SnapshotRetention time.Duration
// L0Retention is how long to keep L0 files after compaction into L1.
// Set to 0 to delete immediately after compaction.
L0Retention time.Duration
tempDirOnce sync.Once
tempDir string
tempDirErr error
@@ -116,6 +136,7 @@ func (vfs *VFS) openMainDB(name string, flags sqlite3vfs.OpenFlag) (sqlite3vfs.F
f := NewVFSFile(vfs.client, name, vfs.logger.With("name", name))
f.PollInterval = vfs.PollInterval
f.CacheSize = vfs.CacheSize
f.vfs = vfs // Store reference to parent VFS for config access
// Initialize write support if enabled
if vfs.WriteEnabled {
@@ -137,6 +158,12 @@ func (vfs *VFS) openMainDB(name string, flags sqlite3vfs.OpenFlag) (sqlite3vfs.F
}
f.bufferPath = filepath.Join(dir, "write-buffer")
}
// Initialize compaction if enabled
if vfs.CompactionEnabled {
f.compactor = NewCompactor(vfs.client, f.logger)
// VFS has no local files, so leave LocalFileOpener/LocalFileDeleter nil
}
}
// Initialize hydration support if enabled
@@ -502,6 +529,13 @@ type VFSFile struct {
PollInterval time.Duration
CacheSize int
// Compaction support (only used when VFS.CompactionEnabled is true)
vfs *VFS // Reference back to parent VFS for config
compactor *Compactor // Shared compaction logic
compactionWg sync.WaitGroup
compactionCtx context.Context
compactionCancel context.CancelFunc
}
// Hydrator handles background hydration of the database to a local file.
@@ -920,6 +954,11 @@ func (f *VFSFile) Open() error {
go func() { defer f.wg.Done(); f.syncLoop() }()
}
// Start compaction monitors if enabled
if f.compactor != nil && f.vfs != nil {
f.startCompactionMonitors()
}
return nil
}
@@ -969,6 +1008,11 @@ func (f *VFSFile) openNewDatabase() error {
go func() { defer f.wg.Done(); f.syncLoop() }()
}
// Start compaction monitors if enabled
if f.compactor != nil && f.vfs != nil {
f.startCompactionMonitors()
}
return nil
}
@@ -1166,6 +1210,12 @@ func (f *VFSFile) Close() error {
f.syncTicker.Stop()
}
// Stop compaction monitors if running
if f.compactionCancel != nil {
f.compactionCancel()
f.compactionWg.Wait()
}
// Final sync of dirty pages before closing
if f.writeEnabled && len(f.dirty) > 0 {
if err := f.syncToRemote(); err != nil {
@@ -2328,3 +2378,223 @@ func lookupVFSFile(fileID uint64) (*VFSFile, bool) {
vfsFile, ok := file.(*VFSFile)
return vfsFile, ok
}
// startCompactionMonitors starts background goroutines for compaction and snapshots.
func (f *VFSFile) startCompactionMonitors() {
f.compactionCtx, f.compactionCancel = context.WithCancel(f.ctx)
// Use configured levels or defaults
levels := f.vfs.CompactionLevels
if levels == nil {
levels = DefaultCompactionLevels
}
// Start compaction monitors for each level
for _, lvl := range levels {
if lvl.Level == 0 {
continue // L0 doesn't need compaction (source level)
}
f.compactionWg.Add(1)
go func(level *CompactionLevel) {
defer f.compactionWg.Done()
f.monitorCompaction(f.compactionCtx, level)
}(lvl)
}
// Start snapshot monitor if configured
if f.vfs.SnapshotInterval > 0 {
f.compactionWg.Add(1)
go func() {
defer f.compactionWg.Done()
f.monitorSnapshots(f.compactionCtx)
}()
}
// Start L0 retention monitor if configured
if f.vfs.L0Retention > 0 {
f.compactionWg.Add(1)
go func() {
defer f.compactionWg.Done()
f.monitorL0Retention(f.compactionCtx)
}()
}
f.logger.Info("compaction monitors started",
"levels", len(levels),
"snapshotInterval", f.vfs.SnapshotInterval,
"l0Retention", f.vfs.L0Retention)
}
// Compact compacts source level files into the destination level.
// Returns ErrNoCompaction if there are no files to compact.
func (f *VFSFile) Compact(ctx context.Context, level int) (*ltx.FileInfo, error) {
if f.compactor == nil {
return nil, fmt.Errorf("compaction not enabled")
}
return f.compactor.Compact(ctx, level)
}
// Snapshot creates a full database snapshot from remote LTX files.
// Unlike DB.Snapshot(), this reads from remote rather than local WAL.
func (f *VFSFile) Snapshot(ctx context.Context) (*ltx.FileInfo, error) {
if f.compactor == nil {
return nil, fmt.Errorf("compaction not enabled")
}
f.mu.Lock()
pageSize := f.pageSize
commit := f.commit
pos := f.pos
pages := make(map[uint32]ltx.PageIndexElem, len(f.index))
for pgno, elem := range f.index {
pages[pgno] = elem
}
f.mu.Unlock()
if pageSize == 0 {
return nil, fmt.Errorf("page size not initialized")
}
// Sort page numbers for consistent output
pgnos := make([]uint32, 0, len(pages))
for pgno := range pages {
pgnos = append(pgnos, pgno)
}
slices.Sort(pgnos)
// Stream snapshot creation
pr, pw := io.Pipe()
go func() {
enc, err := ltx.NewEncoder(pw)
if err != nil {
pw.CloseWithError(err)
return
}
if err := enc.EncodeHeader(ltx.Header{
Version: ltx.Version,
Flags: ltx.HeaderFlagNoChecksum,
PageSize: pageSize,
Commit: commit,
MinTXID: 1,
MaxTXID: pos.TXID,
Timestamp: time.Now().UnixMilli(),
}); err != nil {
pw.CloseWithError(fmt.Errorf("encode header: %w", err))
return
}
for _, pgno := range pgnos {
elem := pages[pgno]
_, data, err := FetchPage(ctx, f.client, elem.Level, elem.MinTXID, elem.MaxTXID, elem.Offset, elem.Size)
if err != nil {
pw.CloseWithError(fmt.Errorf("fetch page %d: %w", pgno, err))
return
}
if err := enc.EncodePage(ltx.PageHeader{Pgno: pgno}, data); err != nil {
pw.CloseWithError(fmt.Errorf("encode page %d: %w", pgno, err))
return
}
}
if err := enc.Close(); err != nil {
pw.CloseWithError(fmt.Errorf("close encoder: %w", err))
return
}
pw.Close()
}()
return f.client.WriteLTXFile(ctx, SnapshotLevel, 1, pos.TXID, pr)
}
// monitorCompaction runs periodic compaction for a level.
func (f *VFSFile) monitorCompaction(ctx context.Context, lvl *CompactionLevel) {
f.logger.Info("starting VFS compaction monitor", "level", lvl.Level, "interval", lvl.Interval)
ticker := time.NewTicker(lvl.Interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
info, err := f.Compact(ctx, lvl.Level)
if err != nil {
if !errors.Is(err, ErrNoCompaction) &&
!errors.Is(err, context.Canceled) &&
!errors.Is(err, context.DeadlineExceeded) {
f.logger.Error("compaction failed", "level", lvl.Level, "error", err)
}
} else {
f.logger.Debug("compaction completed",
"level", lvl.Level,
"minTXID", info.MinTXID,
"maxTXID", info.MaxTXID,
"size", info.Size)
}
}
}
}
// monitorSnapshots runs periodic snapshot creation.
func (f *VFSFile) monitorSnapshots(ctx context.Context) {
f.logger.Info("starting VFS snapshot monitor", "interval", f.vfs.SnapshotInterval)
ticker := time.NewTicker(f.vfs.SnapshotInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
info, err := f.Snapshot(ctx)
if err != nil {
if !errors.Is(err, context.Canceled) &&
!errors.Is(err, context.DeadlineExceeded) {
f.logger.Error("snapshot failed", "error", err)
}
} else {
f.logger.Debug("snapshot created",
"maxTXID", info.MaxTXID,
"size", info.Size)
// Enforce snapshot retention after creating new snapshot
if f.vfs.SnapshotRetention > 0 {
if _, err := f.compactor.EnforceSnapshotRetention(ctx, f.vfs.SnapshotRetention); err != nil {
f.logger.Error("snapshot retention failed", "error", err)
}
}
}
}
}
}
// monitorL0Retention runs periodic L0 retention enforcement.
func (f *VFSFile) monitorL0Retention(ctx context.Context) {
f.logger.Info("starting VFS L0 retention monitor", "retention", f.vfs.L0Retention)
// Check more frequently than the retention period
checkInterval := f.vfs.L0Retention / 4
if checkInterval < time.Minute {
checkInterval = time.Minute
}
ticker := time.NewTicker(checkInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := f.compactor.EnforceL0Retention(ctx, f.vfs.L0Retention); err != nil {
if !errors.Is(err, context.Canceled) &&
!errors.Is(err, context.DeadlineExceeded) {
f.logger.Error("L0 retention enforcement failed", "error", err)
}
}
}
}
}

192
vfs_compaction_test.go Normal file
View File

@@ -0,0 +1,192 @@
//go:build vfs
// +build vfs
package litestream_test
import (
"context"
"log/slog"
"testing"
"time"
"github.com/benbjohnson/litestream"
"github.com/benbjohnson/litestream/file"
)
func TestVFSFile_Compact(t *testing.T) {
t.Run("ManualCompact", func(t *testing.T) {
dir := t.TempDir()
client := file.NewReplicaClient(dir)
// Pre-create some L0 files to test compaction
createTestLTXFile(t, client, 0, 1, 1)
createTestLTXFile(t, client, 0, 2, 2)
createTestLTXFile(t, client, 0, 3, 3)
// Create VFS with compaction enabled
vfs := litestream.NewVFS(client, slog.Default())
vfs.WriteEnabled = true
vfs.CompactionEnabled = true
// Create VFSFile directly to test Compact method
f := litestream.NewVFSFile(client, "test.db", slog.Default())
f.PollInterval = time.Second
f.CacheSize = litestream.DefaultCacheSize
// Initialize the compactor manually
compactor := litestream.NewCompactor(client, slog.Default())
// Compact L0 to L1
info, err := compactor.Compact(context.Background(), 1)
if err != nil {
t.Fatal(err)
}
if info.Level != 1 {
t.Errorf("Level=%d, want 1", info.Level)
}
if info.MinTXID != 1 || info.MaxTXID != 3 {
t.Errorf("TXID range=%d-%d, want 1-3", info.MinTXID, info.MaxTXID)
}
})
}
func TestVFSFile_Snapshot(t *testing.T) {
t.Run("MultiLevelCompaction", func(t *testing.T) {
dir := t.TempDir()
client := file.NewReplicaClient(dir)
// Pre-create L0 files to simulate VFS writes
createTestLTXFile(t, client, 0, 1, 1)
createTestLTXFile(t, client, 0, 2, 2)
createTestLTXFile(t, client, 0, 3, 3)
compactor := litestream.NewCompactor(client, slog.Default())
// Compact L0 to L1
info, err := compactor.Compact(context.Background(), 1)
if err != nil {
t.Fatal(err)
}
if info.Level != 1 {
t.Errorf("Level=%d, want 1", info.Level)
}
t.Logf("Compacted to L1: minTXID=%d, maxTXID=%d", info.MinTXID, info.MaxTXID)
// Compact L1 to L2
info, err = compactor.Compact(context.Background(), 2)
if err != nil {
t.Fatal(err)
}
if info.Level != 2 {
t.Errorf("Level=%d, want 2", info.Level)
}
t.Logf("Compacted to L2: minTXID=%d, maxTXID=%d", info.MinTXID, info.MaxTXID)
// Verify L2 file exists
itr, err := client.LTXFiles(context.Background(), 2, 0, false)
if err != nil {
t.Fatal(err)
}
defer itr.Close()
var l2Count int
for itr.Next() {
l2Count++
}
if l2Count != 1 {
t.Errorf("L2 file count=%d, want 1", l2Count)
}
})
}
func TestDefaultCompactionLevels(t *testing.T) {
levels := litestream.DefaultCompactionLevels
if len(levels) != 4 {
t.Fatalf("DefaultCompactionLevels length=%d, want 4", len(levels))
}
// Verify L0 (raw files, no interval)
if levels[0].Level != 0 {
t.Errorf("levels[0].Level=%d, want 0", levels[0].Level)
}
if levels[0].Interval != 0 {
t.Errorf("levels[0].Interval=%v, want 0", levels[0].Interval)
}
// Verify L1 (30 second compaction)
if levels[1].Level != 1 {
t.Errorf("levels[1].Level=%d, want 1", levels[1].Level)
}
if levels[1].Interval != 30*time.Second {
t.Errorf("levels[1].Interval=%v, want 30s", levels[1].Interval)
}
// Verify L2 (5 minute compaction)
if levels[2].Level != 2 {
t.Errorf("levels[2].Level=%d, want 2", levels[2].Level)
}
if levels[2].Interval != 5*time.Minute {
t.Errorf("levels[2].Interval=%v, want 5m", levels[2].Interval)
}
// Verify L3 (hourly compaction)
if levels[3].Level != 3 {
t.Errorf("levels[3].Level=%d, want 3", levels[3].Level)
}
if levels[3].Interval != time.Hour {
t.Errorf("levels[3].Interval=%v, want 1h", levels[3].Interval)
}
// Verify they validate
if err := levels.Validate(); err != nil {
t.Errorf("DefaultCompactionLevels.Validate()=%v, want nil", err)
}
}
func TestVFS_CompactionConfig(t *testing.T) {
t.Run("DefaultConfig", func(t *testing.T) {
client := file.NewReplicaClient(t.TempDir())
vfs := litestream.NewVFS(client, slog.Default())
// Default should have compaction disabled
if vfs.CompactionEnabled {
t.Error("CompactionEnabled should be false by default")
}
if vfs.CompactionLevels != nil {
t.Error("CompactionLevels should be nil by default")
}
if vfs.SnapshotInterval != 0 {
t.Error("SnapshotInterval should be 0 by default")
}
})
t.Run("WithCompactionConfig", func(t *testing.T) {
client := file.NewReplicaClient(t.TempDir())
vfs := litestream.NewVFS(client, slog.Default())
vfs.WriteEnabled = true
vfs.CompactionEnabled = true
vfs.CompactionLevels = litestream.CompactionLevels{
{Level: 0, Interval: 0},
{Level: 1, Interval: time.Minute},
}
vfs.SnapshotInterval = 24 * time.Hour
vfs.SnapshotRetention = 7 * 24 * time.Hour
vfs.L0Retention = 5 * time.Minute
if !vfs.CompactionEnabled {
t.Error("CompactionEnabled should be true")
}
if len(vfs.CompactionLevels) != 2 {
t.Errorf("CompactionLevels length=%d, want 2", len(vfs.CompactionLevels))
}
if vfs.SnapshotInterval != 24*time.Hour {
t.Errorf("SnapshotInterval=%v, want 24h", vfs.SnapshotInterval)
}
if vfs.SnapshotRetention != 7*24*time.Hour {
t.Errorf("SnapshotRetention=%v, want 168h", vfs.SnapshotRetention)
}
if vfs.L0Retention != 5*time.Minute {
t.Errorf("L0Retention=%v, want 5m", vfs.L0Retention)
}
})
}