docs(examples): add library usage examples (#921)

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-25 05:06:30 +00:00 · 2025-12-24 06:49:42 -07:00
parent 0a441d06fe
commit 3480067f4b
5 changed files with 635 additions and 0 deletions
--- a/.github/workflows/commit.yml
+++ b/.github/workflows/commit.yml
@@ -107,6 +107,10 @@ jobs:

      - run: go install ./cmd/litestream

+      - run: go build -o /dev/null ./_examples/library/basic
+      - run: go build -o /dev/null ./_examples/library/s3
+      - run: go test -v ./_examples/library
+
      - run: go test -v .
      - run: go test -v ./internal
      - run: go test -v ./abs
--- a/_examples/library/README.md
+++ b/_examples/library/README.md
@@ -0,0 +1,162 @@
+# Litestream Library Usage Examples
+
+These examples demonstrate how to use Litestream as a Go library instead of as a
+standalone CLI tool.
+
+## API Stability Warning
+
+The Litestream library API is not considered stable and may change between
+versions. The CLI interface is more stable for production use. Use the library
+API at your own risk, and pin to specific versions.
+
+Note (macOS): macOS uses per-process locks for SQLite, not per-handle locks.
+If you open the same database with two different SQLite driver implementations
+in the same process and close one of them, you can hit locking issues. Prefer
+using the same driver for your app and Litestream (these examples use
+`modernc.org/sqlite`).
+
+## Examples
+
+### Basic (File Backend)
+
+The simplest example using local filesystem replication.
+
+```bash
+cd basic
+go run main.go
+```
+
+This creates:
+
+- `myapp.db` - The SQLite database
+- `replica/` - Directory containing replicated LTX files
+
+### S3 Backend
+
+A more complete example showing the restore-on-startup pattern with S3.
+
+```bash
+cd s3
+
+# Set required environment variables
+export AWS_ACCESS_KEY_ID="your-access-key"
+export AWS_SECRET_ACCESS_KEY="your-secret-key"
+export LITESTREAM_BUCKET="your-bucket-name"
+export LITESTREAM_PATH="databases/myapp"  # optional, defaults to "litestream"
+export AWS_REGION="us-east-1"             # optional, defaults to "us-east-1"
+
+go run main.go
+```
+
+This example:
+
+1. Checks if the local database exists
+2. If not, attempts to restore from S3
+3. Starts background replication to S3
+4. Inserts sample data every 2 seconds
+5. Gracefully shuts down on Ctrl+C
+
+## Core API Pattern
+
+```go
+import (
+    "context"
+    "database/sql"
+    "github.com/benbjohnson/litestream"
+    "github.com/benbjohnson/litestream/file"  // or s3, gs, abs, etc.
+    _ "modernc.org/sqlite"
+)
+
+// 1. Create database wrapper
+db := litestream.NewDB("/path/to/db.sqlite")
+
+// 2. Create replica client
+client := file.NewReplicaClient("/path/to/replica")
+// OR from URL:
+// client, _ := litestream.NewReplicaClientFromURL("s3://bucket/path")
+
+// 3. Attach replica to database
+replica := litestream.NewReplicaWithClient(db, client)
+db.Replica = replica
+client.Replica = replica // file backend only; preserves ownership/permissions
+
+// 4. Create compaction levels (L0 required, plus at least one more)
+levels := litestream.CompactionLevels{
+    {Level: 0},
+    {Level: 1, Interval: 10 * time.Second},
+}
+
+// 5. Create Store to manage DB and background compaction
+store := litestream.NewStore([]*litestream.DB{db}, levels)
+
+// 6. Open Store (opens all DBs, starts background monitors)
+if err := store.Open(ctx); err != nil { ... }
+defer store.Close(context.Background())
+
+// 7. Open your app's SQLite connection for normal database operations
+sqlDB, err := sql.Open("sqlite", "/path/to/db.sqlite")
+if err != nil { ... }
+if _, err := sqlDB.ExecContext(ctx, `PRAGMA journal_mode = wal;`); err != nil { ... }
+if _, err := sqlDB.ExecContext(ctx, `PRAGMA busy_timeout = 5000;`); err != nil { ... }
+```
+
+## Restore Pattern
+
+```go
+// Create replica without database for restore
+replica := litestream.NewReplicaWithClient(nil, client)
+
+opt := litestream.NewRestoreOptions()
+opt.OutputPath = "/path/to/restored.db"
+// Optional: point-in-time restore
+// opt.Timestamp = time.Now().Add(-1 * time.Hour)
+
+if err := replica.Restore(ctx, opt); err != nil {
+    if errors.Is(err, litestream.ErrTxNotAvailable) || errors.Is(err, litestream.ErrNoSnapshots) {
+        // No backup available, create fresh database
+    }
+    return err
+}
+```
+
+## Supported Backends
+
+- `file` - Local filesystem
+- `s3` - AWS S3 and S3-compatible storage
+- `gs` - Google Cloud Storage
+- `abs` - Azure Blob Storage
+- `oss` - Alibaba Cloud OSS
+- `sftp` - SFTP servers
+- `nats` - NATS JetStream
+- `webdav` - WebDAV servers
+
+## Key Configuration Options
+
+### Store Settings
+
+```go
+store.SnapshotInterval  = 24 * time.Hour  // How often to create snapshots
+store.SnapshotRetention = 24 * time.Hour  // How long to keep snapshots
+store.L0Retention       = 5 * time.Minute // How long to keep L0 files after compaction
+```
+
+### DB Settings
+
+```go
+db.MonitorInterval    = 1 * time.Second   // How often to check for changes
+db.CheckpointInterval = 1 * time.Minute   // Time-based checkpoint interval
+db.MinCheckpointPageN = 1000              // Page threshold for checkpoint
+db.BusyTimeout        = 1 * time.Second   // SQLite busy timeout
+```
+
+### Replica Settings
+
+```go
+replica.SyncInterval   = 1 * time.Second  // Time between syncs
+replica.MonitorEnabled = true             // Auto-sync in background
+```
+
+## Resources
+
+- [Litestream Documentation](https://litestream.io)
+- [GitHub Repository](https://github.com/benbjohnson/litestream)
--- a/_examples/library/basic/main.go
+++ b/_examples/library/basic/main.go
@@ -0,0 +1,139 @@
+// Example: Basic Litestream Library Usage
+//
+// This example demonstrates the simplest way to use Litestream as a Go library.
+// It replicates a SQLite database to the local filesystem.
+//
+// Run: go run main.go
+package main
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	_ "modernc.org/sqlite"
+
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/file"
+)
+
+func main() {
+	if err := run(context.Background()); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func run(ctx context.Context) error {
+	// Paths for this example
+	dbPath := "./myapp.db"
+	replicaPath := "./replica"
+
+	// 1. Create the Litestream DB wrapper
+	db := litestream.NewDB(dbPath)
+
+	// 2. Create a replica client (file-based for this example)
+	client := file.NewReplicaClient(replicaPath)
+
+	// 3. Create a replica and attach it to the database
+	replica := litestream.NewReplicaWithClient(db, client)
+	db.Replica = replica
+	client.Replica = replica
+
+	// 4. Create compaction levels (L0 is required, plus at least one more level)
+	levels := litestream.CompactionLevels{
+		{Level: 0},
+		{Level: 1, Interval: 10 * time.Second},
+	}
+
+	// 5. Create a Store to manage the database and background compaction
+	store := litestream.NewStore([]*litestream.DB{db}, levels)
+
+	// 6. Open the store (opens all DBs and starts background monitors)
+	if err := store.Open(ctx); err != nil {
+		return fmt.Errorf("open store: %w", err)
+	}
+	defer func() {
+		if err := store.Close(context.Background()); err != nil {
+			log.Printf("close store: %v", err)
+		}
+	}()
+
+	// 7. Open your app's SQLite connection for normal database operations
+	sqlDB, err := openAppDB(ctx, dbPath)
+	if err != nil {
+		return fmt.Errorf("open app db: %w", err)
+	}
+	defer sqlDB.Close()
+	if err := initSchema(ctx, sqlDB); err != nil {
+		return fmt.Errorf("init schema: %w", err)
+	}
+
+	// Insert some test data periodically
+	ticker := time.NewTicker(2 * time.Second)
+	defer ticker.Stop()
+
+	// Handle shutdown gracefully
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+
+	fmt.Println("Writing data every 2 seconds. Press Ctrl+C to stop.")
+	fmt.Printf("Database: %s\n", dbPath)
+	fmt.Printf("Replica:  %s\n", replicaPath)
+
+	for {
+		select {
+		case <-ticker.C:
+			if err := insertRow(ctx, sqlDB); err != nil {
+				log.Printf("insert row: %v", err)
+			}
+		case <-sigCh:
+			fmt.Println("\nShutting down...")
+			return nil
+		}
+	}
+}
+
+func openAppDB(ctx context.Context, path string) (*sql.DB, error) {
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := db.ExecContext(ctx, `PRAGMA journal_mode = wal;`); err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+	if _, err := db.ExecContext(ctx, `PRAGMA busy_timeout = 5000;`); err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+	return db, nil
+}
+
+func initSchema(ctx context.Context, db *sql.DB) error {
+	_, err := db.ExecContext(ctx, `
+		CREATE TABLE IF NOT EXISTS events (
+			id INTEGER PRIMARY KEY AUTOINCREMENT,
+			message TEXT NOT NULL,
+			created_at TEXT NOT NULL
+		)
+	`)
+	return err
+}
+
+func insertRow(ctx context.Context, db *sql.DB) error {
+	msg := fmt.Sprintf("Event at %s", time.Now().Format(time.RFC3339))
+	result, err := db.ExecContext(ctx,
+		`INSERT INTO events (message, created_at) VALUES (?, ?)`,
+		msg, time.Now().Format(time.RFC3339))
+	if err != nil {
+		return err
+	}
+	id, _ := result.LastInsertId()
+	fmt.Printf("Inserted row %d: %s\n", id, msg)
+	return nil
+}
--- a/_examples/library/library_example_test.go
+++ b/_examples/library/library_example_test.go
@@ -0,0 +1,121 @@
+package library_test
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"testing"
+	"time"
+
+	_ "modernc.org/sqlite"
+
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/file"
+)
+
+func TestLibraryExampleFileBackend(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	rootDir := t.TempDir()
+	dbPath := filepath.Join(rootDir, "example.db")
+	replicaPath := filepath.Join(rootDir, "replica")
+
+	db := litestream.NewDB(dbPath)
+	client := file.NewReplicaClient(replicaPath)
+	replica := litestream.NewReplicaWithClient(db, client)
+	db.Replica = replica
+	client.Replica = replica
+
+	levels := litestream.CompactionLevels{
+		{Level: 0},
+		{Level: 1, Interval: 10 * time.Second},
+	}
+	store := litestream.NewStore([]*litestream.DB{db}, levels)
+
+	closed := false
+	t.Cleanup(func() {
+		if !closed {
+			_ = store.Close(context.Background())
+		}
+	})
+
+	if err := store.Open(ctx); err != nil {
+		t.Fatalf("open store: %v", err)
+	}
+
+	sqlDB, err := openAppDB(ctx, dbPath)
+	if err != nil {
+		t.Fatalf("open app db: %v", err)
+	}
+	defer sqlDB.Close()
+
+	if _, err := sqlDB.ExecContext(ctx, `
+		CREATE TABLE IF NOT EXISTS events (
+			id INTEGER PRIMARY KEY AUTOINCREMENT,
+			message TEXT NOT NULL
+		)
+	`); err != nil {
+		t.Fatalf("create table: %v", err)
+	}
+
+	if _, err := sqlDB.ExecContext(ctx, `INSERT INTO events (message) VALUES ('hello');`); err != nil {
+		t.Fatalf("insert row: %v", err)
+	}
+
+	if err := waitForLTXFiles(replicaPath, 5*time.Second); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := sqlDB.Close(); err != nil {
+		t.Fatalf("close app db: %v", err)
+	}
+
+	if err := store.Close(ctx); err != nil && !errors.Is(err, sql.ErrTxDone) {
+		t.Fatalf("close store: %v", err)
+	}
+	closed = true
+
+	restoreClient := file.NewReplicaClient(replicaPath)
+	restoreReplica := litestream.NewReplicaWithClient(nil, restoreClient)
+
+	restorePath := filepath.Join(rootDir, "restored.db")
+	opt := litestream.NewRestoreOptions()
+	opt.OutputPath = restorePath
+	if err := restoreReplica.Restore(ctx, opt); err != nil {
+		t.Fatalf("restore: %v", err)
+	}
+}
+
+func openAppDB(ctx context.Context, path string) (*sql.DB, error) {
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := db.ExecContext(ctx, `PRAGMA journal_mode = wal;`); err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+	if _, err := db.ExecContext(ctx, `PRAGMA busy_timeout = 5000;`); err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+	return db, nil
+}
+
+func waitForLTXFiles(replicaPath string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		matches, err := filepath.Glob(filepath.Join(replicaPath, "ltx", "0", "*.ltx"))
+		if err != nil {
+			return fmt.Errorf("glob ltx files: %w", err)
+		}
+		if len(matches) > 0 {
+			return nil
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return fmt.Errorf("timeout waiting for ltx files in %s", replicaPath)
+}
--- a/_examples/library/s3/main.go
+++ b/_examples/library/s3/main.go
@@ -0,0 +1,209 @@
+// Example: Litestream Library Usage with S3 and Restore-on-Startup
+//
+// This example demonstrates a production-like pattern for using Litestream:
+// - Check if local database exists
+// - If not, restore from S3 backup (if available)
+// - Start replication to S3
+// - Graceful shutdown
+//
+// Environment variables:
+//   - AWS_ACCESS_KEY_ID: AWS access key
+//   - AWS_SECRET_ACCESS_KEY: AWS secret key
+//   - LITESTREAM_BUCKET: S3 bucket name (e.g., "my-backup-bucket")
+//   - LITESTREAM_PATH: Path within bucket (e.g., "databases/myapp")
+//   - AWS_REGION: AWS region (default: us-east-1)
+//
+// Run: go run main.go
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	_ "modernc.org/sqlite"
+
+	"github.com/benbjohnson/litestream"
+	"github.com/benbjohnson/litestream/s3"
+)
+
+const dbPath = "./myapp.db"
+
+func main() {
+	if err := run(context.Background()); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func run(ctx context.Context) error {
+	// Load configuration from environment
+	bucket := os.Getenv("LITESTREAM_BUCKET")
+	if bucket == "" {
+		return fmt.Errorf("LITESTREAM_BUCKET environment variable required")
+	}
+	path := os.Getenv("LITESTREAM_PATH")
+	if path == "" {
+		path = "litestream"
+	}
+	region := os.Getenv("AWS_REGION")
+	if region == "" {
+		region = "us-east-1"
+	}
+
+	// 1. Create S3 replica client
+	client := s3.NewReplicaClient()
+	client.Bucket = bucket
+	client.Path = path
+	client.Region = region
+	client.AccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID")
+	client.SecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
+
+	// 2. Restore from S3 if local database doesn't exist
+	if err := restoreIfNotExists(ctx, client, dbPath); err != nil {
+		return fmt.Errorf("restore: %w", err)
+	}
+
+	// 3. Create the Litestream DB wrapper
+	db := litestream.NewDB(dbPath)
+
+	// 4. Create replica and attach to database
+	replica := litestream.NewReplicaWithClient(db, client)
+	db.Replica = replica
+
+	// 5. Create compaction levels (L0 is required, plus at least one more level)
+	levels := litestream.CompactionLevels{
+		{Level: 0},
+		{Level: 1, Interval: 10 * time.Second},
+	}
+
+	// 6. Create a Store to manage the database and background compaction
+	store := litestream.NewStore([]*litestream.DB{db}, levels)
+
+	// 7. Open store (opens all DBs and starts background monitors)
+	if err := store.Open(ctx); err != nil {
+		return fmt.Errorf("open store: %w", err)
+	}
+	defer func() {
+		log.Println("Closing store...")
+		if err := store.Close(context.Background()); err != nil {
+			log.Printf("close store: %v", err)
+		}
+	}()
+
+	// 8. Open your app's SQLite connection for normal operations
+	sqlDB, err := openAppDB(ctx, dbPath)
+	if err != nil {
+		return fmt.Errorf("open app db: %w", err)
+	}
+	defer sqlDB.Close()
+	if err := initSchema(ctx, sqlDB); err != nil {
+		return fmt.Errorf("init schema: %w", err)
+	}
+
+	// Start the application
+	log.Printf("Database: %s", dbPath)
+	log.Printf("Replicating to: s3://%s/%s", bucket, path)
+	log.Println("Writing data every 2 seconds. Press Ctrl+C to stop.")
+
+	ticker := time.NewTicker(2 * time.Second)
+	defer ticker.Stop()
+
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+
+	for {
+		select {
+		case <-ticker.C:
+			if err := insertRow(ctx, sqlDB); err != nil {
+				log.Printf("insert row: %v", err)
+			}
+		case <-sigCh:
+			log.Println("Shutting down...")
+			return nil
+		}
+	}
+}
+
+// restoreIfNotExists restores the database from S3 if it doesn't exist locally.
+func restoreIfNotExists(ctx context.Context, client *s3.ReplicaClient, dbPath string) error {
+	// Check if database already exists
+	if _, err := os.Stat(dbPath); err == nil {
+		log.Println("Local database found, skipping restore")
+		return nil
+	} else if !os.IsNotExist(err) {
+		return err
+	}
+
+	log.Println("Local database not found, attempting restore from S3...")
+
+	// Initialize the client
+	if err := client.Init(ctx); err != nil {
+		return fmt.Errorf("init s3 client: %w", err)
+	}
+
+	// Create a replica (without DB) for restore
+	replica := litestream.NewReplicaWithClient(nil, client)
+
+	// Set up restore options
+	opt := litestream.NewRestoreOptions()
+	opt.OutputPath = dbPath
+
+	// Attempt restore
+	if err := replica.Restore(ctx, opt); err != nil {
+		// If no backup exists, that's OK - we'll create a fresh database
+		if errors.Is(err, litestream.ErrTxNotAvailable) || errors.Is(err, litestream.ErrNoSnapshots) {
+			log.Println("No backup found in S3, will create new database")
+			return nil
+		}
+		return err
+	}
+
+	log.Println("Database restored from S3")
+	return nil
+}
+
+func openAppDB(ctx context.Context, path string) (*sql.DB, error) {
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := db.ExecContext(ctx, `PRAGMA journal_mode = wal;`); err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+	if _, err := db.ExecContext(ctx, `PRAGMA busy_timeout = 5000;`); err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+	return db, nil
+}
+
+func initSchema(ctx context.Context, db *sql.DB) error {
+	_, err := db.ExecContext(ctx, `
+		CREATE TABLE IF NOT EXISTS events (
+			id INTEGER PRIMARY KEY AUTOINCREMENT,
+			message TEXT NOT NULL,
+			created_at TEXT NOT NULL
+		)
+	`)
+	return err
+}
+
+func insertRow(ctx context.Context, db *sql.DB) error {
+	msg := fmt.Sprintf("Event at %s", time.Now().Format(time.RFC3339))
+	result, err := db.ExecContext(ctx,
+		`INSERT INTO events (message, created_at) VALUES (?, ?)`,
+		msg, time.Now().Format(time.RFC3339))
+	if err != nil {
+		return err
+	}
+	id, _ := result.LastInsertId()
+	log.Printf("Inserted row %d: %s", id, msg)
+	return nil
+}