feat(sqlite-persistence): prune applied_tx by default with safe replay recovery

.changeset/node-default-applied-tx-pruning.md

@@ -0,0 +1,5 @@
+---
+'@tanstack/node-db-sqlite-persistence': minor
+---
+
+`createNodeSQLitePersistence` now prunes the `applied_tx` log by default so the SQLite file no longer grows without bound. When prune options are omitted, the node driver applies `appliedTxPruneMaxRows: 1_000` and `appliedTxPruneMaxAgeSeconds: 86_400` (24h). Both remain overridable, and passing `0` disables that limit. The defaults are exported as `DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS` and `DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS`.

packages/node-db-sqlite-persistence/README.md

@@ -47,3 +47,27 @@ export const todosCollection = createCollection(
   mode-specific behavior (`sync-present` vs `sync-absent`) automatically.
 - `schemaVersion` is specified per collection via `persistedCollectionOptions`.
 - Call `database.close()` when your app shuts down.
+
+## Applied transaction pruning
+
+The `applied_tx` log is a replayable cache, so it is pruned by default to keep
+the SQLite file from growing without bound. When you don't pass prune options,
+the node driver applies:
+
+- `appliedTxPruneMaxRows: 1_000` (per-collection row cap)
+- `appliedTxPruneMaxAgeSeconds: 86_400` (24h age backstop)
+
+Pruning runs inside each write transaction, so every collection self-trims on
+its next sync. Override either value to tune retention, or pass `0` to disable
+that limit:
+
+```ts
+const persistence = createNodeSQLitePersistence({
+  database,
+  appliedTxPruneMaxRows: 5_000, // higher row cap
+  appliedTxPruneMaxAgeSeconds: 0, // disable the age backstop
+})
+```
+
+The defaults are exported as `DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS` and
+`DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS`.

packages/node-db-sqlite-persistence/src/index.ts

@@ -1,4 +1,8 @@
-export { createNodeSQLitePersistence } from './node-persistence'
+export {
+  createNodeSQLitePersistence,
+  DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
+  DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
+} from './node-persistence'
 export type {
   BetterSqlite3Database,
   NodeSQLitePersistenceOptions,

packages/node-db-sqlite-persistence/src/node-persistence.ts

@@ -35,6 +35,20 @@ type NodeSQLitePersistenceBaseOptions = Omit<
 
 export type NodeSQLitePersistenceOptions = NodeSQLitePersistenceBaseOptions
 
+/**
+ * Default cap on retained `applied_tx` rows per collection. The log is a
+ * replayable cache, so a bounded row count keeps the SQLite file from growing
+ * without limit. Pass `appliedTxPruneMaxRows: 0` to disable the row cap.
+ */
+export const DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS = 1_000
+
+/**
+ * Default age backstop for retained `applied_tx` rows, in seconds (24h). Rows
+ * older than this are pruned on the next write. Pass
+ * `appliedTxPruneMaxAgeSeconds: 0` to disable the age backstop.
+ */
+export const DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS = 24 * 60 * 60
+
 function normalizeSchemaMismatchPolicy(
   policy: NodeSQLiteSchemaMismatchPolicy,
 ): NodeSQLiteCoreSchemaMismatchPolicy {
@@ -81,8 +95,11 @@ function resolveAdapterBaseOptions(
   `driver` | `schemaVersion` | `schemaMismatchPolicy`
 > {
   return {
-    appliedTxPruneMaxRows: options.appliedTxPruneMaxRows,
-    appliedTxPruneMaxAgeSeconds: options.appliedTxPruneMaxAgeSeconds,
+    appliedTxPruneMaxRows:
+      options.appliedTxPruneMaxRows ?? DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
+    appliedTxPruneMaxAgeSeconds:
+      options.appliedTxPruneMaxAgeSeconds ??
+      DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
     pullSinceReloadThreshold: options.pullSinceReloadThreshold,
   }
 }

packages/node-db-sqlite-persistence/tests/node-persistence.test.ts

@@ -127,6 +127,121 @@ describe(`node persistence helpers`, () => {
     }
   })
 
+  it(`prunes applied_tx rows past the default age backstop`, async () => {
+    const tempDirectory = mkdtempSync(join(tmpdir(), `db-node-default-prune-`))
+    const dbPath = join(tempDirectory, `state.sqlite`)
+    const collectionId = `default-prune`
+    const database = new BetterSqlite3(dbPath)
+
+    try {
+      const persistence = createNodeSQLitePersistence({ database })
+
+      await persistence.adapter.applyCommittedTx(collectionId, {
+        txId: `tx-1`,
+        term: 1,
+        seq: 1,
+        rowVersion: 1,
+        mutations: [
+          {
+            type: `insert`,
+            key: `1`,
+            value: { id: `1`, title: `old`, score: 1 },
+          },
+        ],
+      })
+
+      // Backdate the first row well beyond the 24h default age backstop.
+      database
+        .prepare(
+          `UPDATE applied_tx SET applied_at = 0 WHERE collection_id = ? AND seq = 1`,
+        )
+        .run(collectionId)
+
+      await persistence.adapter.applyCommittedTx(collectionId, {
+        txId: `tx-2`,
+        term: 1,
+        seq: 2,
+        rowVersion: 2,
+        mutations: [
+          {
+            type: `insert`,
+            key: `2`,
+            value: { id: `2`, title: `new`, score: 2 },
+          },
+        ],
+      })
+
+      const appliedRows = database
+        .prepare(
+          `SELECT seq FROM applied_tx WHERE collection_id = ? ORDER BY seq ASC`,
+        )
+        .all(collectionId) as Array<{ seq: number }>
+      expect(appliedRows.map((row) => row.seq)).toEqual([2])
+    } finally {
+      database.close()
+      rmSync(tempDirectory, { recursive: true, force: true })
+    }
+  })
+
+  it(`leaves applied_tx rows untouched when pruning is disabled`, async () => {
+    const tempDirectory = mkdtempSync(join(tmpdir(), `db-node-no-prune-`))
+    const dbPath = join(tempDirectory, `state.sqlite`)
+    const collectionId = `no-prune`
+    const database = new BetterSqlite3(dbPath)
+
+    try {
+      const persistence = createNodeSQLitePersistence({
+        database,
+        appliedTxPruneMaxRows: 0,
+        appliedTxPruneMaxAgeSeconds: 0,
+      })
+
+      await persistence.adapter.applyCommittedTx(collectionId, {
+        txId: `tx-1`,
+        term: 1,
+        seq: 1,
+        rowVersion: 1,
+        mutations: [
+          {
+            type: `insert`,
+            key: `1`,
+            value: { id: `1`, title: `old`, score: 1 },
+          },
+        ],
+      })
+
+      database
+        .prepare(
+          `UPDATE applied_tx SET applied_at = 0 WHERE collection_id = ? AND seq = 1`,
+        )
+        .run(collectionId)
+
+      await persistence.adapter.applyCommittedTx(collectionId, {
+        txId: `tx-2`,
+        term: 1,
+        seq: 2,
+        rowVersion: 2,
+        mutations: [
+          {
+            type: `insert`,
+            key: `2`,
+            value: { id: `2`, title: `new`, score: 2 },
+          },
+        ],
+      })
+
+      const appliedRows = database
+        .prepare(
+          `SELECT seq FROM applied_tx WHERE collection_id = ? ORDER BY seq ASC`,
+        )
+        .all(collectionId) as Array<{ seq: number }>
+      expect(appliedRows.map((row) => row.seq)).toEqual([1, 2])
+    } finally {
+      database.close()
+      rmSync(tempDirectory, { recursive: true, force: true })
+    }
+  })
+
   it(`infers schema policy from sync mode`, async () => {
     const tempDirectory = mkdtempSync(join(tmpdir(), `db-node-schema-infer-`))
     const dbPath = join(tempDirectory, `state.sqlite`)