Add configurable automatic database backup scheduling

This commit is contained in:
Dotta
2026-03-04 18:03:23 -06:00
parent f6a09bcbea
commit c145074daf
17 changed files with 722 additions and 351 deletions

View File

@@ -21,6 +21,12 @@ function writeBaseConfig(configPath: string) {
mode: "embedded-postgres", mode: "embedded-postgres",
embeddedPostgresDataDir: "/tmp/paperclip-db", embeddedPostgresDataDir: "/tmp/paperclip-db",
embeddedPostgresPort: 54329, embeddedPostgresPort: 54329,
backup: {
enabled: true,
intervalMinutes: 60,
retentionDays: 30,
dir: "/tmp/paperclip-backups",
},
}, },
logging: { logging: {
mode: "file", mode: "file",
@@ -68,4 +74,3 @@ describe("allowed-hostname command", () => {
expect(raw.server.allowedHostnames).toEqual(["dotta-macbook-pro"]); expect(raw.server.allowedHostnames).toEqual(["dotta-macbook-pro"]);
}); });
}); });

View File

@@ -10,6 +10,7 @@ import { defaultSecretsConfig, promptSecrets } from "../prompts/secrets.js";
import { defaultStorageConfig, promptStorage } from "../prompts/storage.js"; import { defaultStorageConfig, promptStorage } from "../prompts/storage.js";
import { promptServer } from "../prompts/server.js"; import { promptServer } from "../prompts/server.js";
import { import {
resolveDefaultBackupDir,
resolveDefaultEmbeddedPostgresDir, resolveDefaultEmbeddedPostgresDir,
resolveDefaultLogsDir, resolveDefaultLogsDir,
resolvePaperclipInstanceId, resolvePaperclipInstanceId,
@@ -39,6 +40,12 @@ function defaultConfig(): PaperclipConfig {
mode: "embedded-postgres", mode: "embedded-postgres",
embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(instanceId), embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(instanceId),
embeddedPostgresPort: 54329, embeddedPostgresPort: 54329,
backup: {
enabled: true,
intervalMinutes: 60,
retentionDays: 30,
dir: resolveDefaultBackupDir(instanceId),
},
}, },
logging: { logging: {
mode: "file", mode: "file",
@@ -118,7 +125,7 @@ export async function configure(opts: {
switch (section) { switch (section) {
case "database": case "database":
config.database = await promptDatabase(); config.database = await promptDatabase(config.database);
break; break;
case "llm": { case "llm": {
const llm = await promptLlm(); const llm = await promptLlm();

View File

@@ -12,6 +12,7 @@ import { defaultStorageConfig, promptStorage } from "../prompts/storage.js";
import { promptServer } from "../prompts/server.js"; import { promptServer } from "../prompts/server.js";
import { import {
describeLocalInstancePaths, describeLocalInstancePaths,
resolveDefaultBackupDir,
resolveDefaultEmbeddedPostgresDir, resolveDefaultEmbeddedPostgresDir,
resolveDefaultLogsDir, resolveDefaultLogsDir,
resolvePaperclipInstanceId, resolvePaperclipInstanceId,
@@ -35,6 +36,12 @@ function quickstartDefaults(): Pick<PaperclipConfig, "database" | "logging" | "s
mode: "embedded-postgres", mode: "embedded-postgres",
embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(instanceId), embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(instanceId),
embeddedPostgresPort: 54329, embeddedPostgresPort: 54329,
backup: {
enabled: true,
intervalMinutes: 60,
retentionDays: 30,
dir: resolveDefaultBackupDir(instanceId),
},
}, },
logging: { logging: {
mode: "file", mode: "file",
@@ -120,7 +127,7 @@ export async function onboard(opts: OnboardOptions): Promise<void> {
if (setupMode === "advanced") { if (setupMode === "advanced") {
p.log.step(pc.bold("Database")); p.log.step(pc.bold("Database"));
database = await promptDatabase(); database = await promptDatabase(database);
if (database.mode === "postgres" && database.connectionString) { if (database.mode === "postgres" && database.connectionString) {
const s = p.spinner(); const s = p.spinner();

View File

@@ -49,6 +49,10 @@ export function resolveDefaultStorageDir(instanceId?: string): string {
return path.resolve(resolvePaperclipInstanceRoot(instanceId), "data", "storage"); return path.resolve(resolvePaperclipInstanceRoot(instanceId), "data", "storage");
} }
export function resolveDefaultBackupDir(instanceId?: string): string {
return path.resolve(resolvePaperclipInstanceRoot(instanceId), "data", "backups");
}
export function expandHomePrefix(value: string): string { export function expandHomePrefix(value: string): string {
if (value === "~") return os.homedir(); if (value === "~") return os.homedir();
if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2)); if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2));
@@ -64,6 +68,7 @@ export function describeLocalInstancePaths(instanceId?: string) {
instanceRoot, instanceRoot,
configPath: resolveDefaultConfigPath(resolvedInstanceId), configPath: resolveDefaultConfigPath(resolvedInstanceId),
embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(resolvedInstanceId), embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(resolvedInstanceId),
backupDir: resolveDefaultBackupDir(resolvedInstanceId),
logDir: resolveDefaultLogsDir(resolvedInstanceId), logDir: resolveDefaultLogsDir(resolvedInstanceId),
secretsKeyFilePath: resolveDefaultSecretsKeyFilePath(resolvedInstanceId), secretsKeyFilePath: resolveDefaultSecretsKeyFilePath(resolvedInstanceId),
storageDir: resolveDefaultStorageDir(resolvedInstanceId), storageDir: resolveDefaultStorageDir(resolvedInstanceId),

View File

@@ -2,6 +2,7 @@ export {
paperclipConfigSchema, paperclipConfigSchema,
configMetaSchema, configMetaSchema,
llmConfigSchema, llmConfigSchema,
databaseBackupConfigSchema,
databaseConfigSchema, databaseConfigSchema,
loggingConfigSchema, loggingConfigSchema,
serverConfigSchema, serverConfigSchema,
@@ -13,6 +14,7 @@ export {
secretsLocalEncryptedConfigSchema, secretsLocalEncryptedConfigSchema,
type PaperclipConfig, type PaperclipConfig,
type LlmConfig, type LlmConfig,
type DatabaseBackupConfig,
type DatabaseConfig, type DatabaseConfig,
type LoggingConfig, type LoggingConfig,
type ServerConfig, type ServerConfig,

View File

@@ -1,9 +1,26 @@
import * as p from "@clack/prompts"; import * as p from "@clack/prompts";
import type { DatabaseConfig } from "../config/schema.js"; import type { DatabaseConfig } from "../config/schema.js";
import { resolveDefaultEmbeddedPostgresDir, resolvePaperclipInstanceId } from "../config/home.js"; import {
resolveDefaultBackupDir,
resolveDefaultEmbeddedPostgresDir,
resolvePaperclipInstanceId,
} from "../config/home.js";
export async function promptDatabase(): Promise<DatabaseConfig> { export async function promptDatabase(current?: DatabaseConfig): Promise<DatabaseConfig> {
const defaultEmbeddedDir = resolveDefaultEmbeddedPostgresDir(resolvePaperclipInstanceId()); const instanceId = resolvePaperclipInstanceId();
const defaultEmbeddedDir = resolveDefaultEmbeddedPostgresDir(instanceId);
const defaultBackupDir = resolveDefaultBackupDir(instanceId);
const base: DatabaseConfig = current ?? {
mode: "embedded-postgres",
embeddedPostgresDataDir: defaultEmbeddedDir,
embeddedPostgresPort: 54329,
backup: {
enabled: true,
intervalMinutes: 60,
retentionDays: 30,
dir: defaultBackupDir,
},
};
const mode = await p.select({ const mode = await p.select({
message: "Database mode", message: "Database mode",
@@ -11,6 +28,7 @@ export async function promptDatabase(): Promise<DatabaseConfig> {
{ value: "embedded-postgres" as const, label: "Embedded PostgreSQL (managed locally)", hint: "recommended" }, { value: "embedded-postgres" as const, label: "Embedded PostgreSQL (managed locally)", hint: "recommended" },
{ value: "postgres" as const, label: "PostgreSQL (external server)" }, { value: "postgres" as const, label: "PostgreSQL (external server)" },
], ],
initialValue: base.mode,
}); });
if (p.isCancel(mode)) { if (p.isCancel(mode)) {
@@ -18,9 +36,14 @@ export async function promptDatabase(): Promise<DatabaseConfig> {
process.exit(0); process.exit(0);
} }
let connectionString: string | undefined = base.connectionString;
let embeddedPostgresDataDir = base.embeddedPostgresDataDir || defaultEmbeddedDir;
let embeddedPostgresPort = base.embeddedPostgresPort || 54329;
if (mode === "postgres") { if (mode === "postgres") {
const connectionString = await p.text({ const value = await p.text({
message: "PostgreSQL connection string", message: "PostgreSQL connection string",
defaultValue: base.connectionString ?? "",
placeholder: "postgres://user:pass@localhost:5432/paperclip", placeholder: "postgres://user:pass@localhost:5432/paperclip",
validate: (val) => { validate: (val) => {
if (!val) return "Connection string is required for PostgreSQL mode"; if (!val) return "Connection string is required for PostgreSQL mode";
@@ -28,48 +51,107 @@ export async function promptDatabase(): Promise<DatabaseConfig> {
}, },
}); });
if (p.isCancel(connectionString)) { if (p.isCancel(value)) {
p.cancel("Setup cancelled."); p.cancel("Setup cancelled.");
process.exit(0); process.exit(0);
} }
return { connectionString = value;
mode: "postgres", } else {
connectionString, const dataDir = await p.text({
embeddedPostgresDataDir: defaultEmbeddedDir, message: "Embedded PostgreSQL data directory",
embeddedPostgresPort: 54329, defaultValue: base.embeddedPostgresDataDir || defaultEmbeddedDir,
}; placeholder: defaultEmbeddedDir,
});
if (p.isCancel(dataDir)) {
p.cancel("Setup cancelled.");
process.exit(0);
}
embeddedPostgresDataDir = dataDir || defaultEmbeddedDir;
const portValue = await p.text({
message: "Embedded PostgreSQL port",
defaultValue: String(base.embeddedPostgresPort || 54329),
placeholder: "54329",
validate: (val) => {
const n = Number(val);
if (!Number.isInteger(n) || n < 1 || n > 65535) return "Port must be an integer between 1 and 65535";
},
});
if (p.isCancel(portValue)) {
p.cancel("Setup cancelled.");
process.exit(0);
}
embeddedPostgresPort = Number(portValue || "54329");
connectionString = undefined;
} }
const embeddedPostgresDataDir = await p.text({ const backupEnabled = await p.confirm({
message: "Embedded PostgreSQL data directory", message: "Enable automatic database backups?",
defaultValue: defaultEmbeddedDir, initialValue: base.backup.enabled,
placeholder: defaultEmbeddedDir,
}); });
if (p.isCancel(backupEnabled)) {
if (p.isCancel(embeddedPostgresDataDir)) {
p.cancel("Setup cancelled."); p.cancel("Setup cancelled.");
process.exit(0); process.exit(0);
} }
const embeddedPostgresPort = await p.text({ const backupDirInput = await p.text({
message: "Embedded PostgreSQL port", message: "Backup directory",
defaultValue: "54329", defaultValue: base.backup.dir || defaultBackupDir,
placeholder: "54329", placeholder: defaultBackupDir,
validate: (val) => (!val || val.trim().length === 0 ? "Backup directory is required" : undefined),
});
if (p.isCancel(backupDirInput)) {
p.cancel("Setup cancelled.");
process.exit(0);
}
const backupIntervalInput = await p.text({
message: "Backup interval (minutes)",
defaultValue: String(base.backup.intervalMinutes || 60),
placeholder: "60",
validate: (val) => { validate: (val) => {
const n = Number(val); const n = Number(val);
if (!Number.isInteger(n) || n < 1 || n > 65535) return "Port must be an integer between 1 and 65535"; if (!Number.isInteger(n) || n < 1) return "Interval must be a positive integer";
if (n > 10080) return "Interval must be 10080 minutes (7 days) or less";
return undefined;
}, },
}); });
if (p.isCancel(backupIntervalInput)) {
p.cancel("Setup cancelled.");
process.exit(0);
}
if (p.isCancel(embeddedPostgresPort)) { const backupRetentionInput = await p.text({
message: "Backup retention (days)",
defaultValue: String(base.backup.retentionDays || 30),
placeholder: "30",
validate: (val) => {
const n = Number(val);
if (!Number.isInteger(n) || n < 1) return "Retention must be a positive integer";
if (n > 3650) return "Retention must be 3650 days or less";
return undefined;
},
});
if (p.isCancel(backupRetentionInput)) {
p.cancel("Setup cancelled."); p.cancel("Setup cancelled.");
process.exit(0); process.exit(0);
} }
return { return {
mode: "embedded-postgres", mode,
embeddedPostgresDataDir: embeddedPostgresDataDir || defaultEmbeddedDir, connectionString,
embeddedPostgresPort: Number(embeddedPostgresPort || "54329"), embeddedPostgresDataDir,
embeddedPostgresPort,
backup: {
enabled: backupEnabled,
intervalMinutes: Number(backupIntervalInput || "60"),
retentionDays: Number(backupRetentionInput || "30"),
dir: backupDirInput || defaultBackupDir,
},
}; };
} }

View File

@@ -141,6 +141,28 @@ pnpm dev
If you set `DATABASE_URL`, the server will use that instead of embedded PostgreSQL. If you set `DATABASE_URL`, the server will use that instead of embedded PostgreSQL.
## Automatic DB Backups
Paperclip can run automatic DB backups on a timer. Defaults:
- enabled
- every 60 minutes
- retain 30 days
- backup dir: `~/.paperclip/instances/default/data/backups`
Configure these in:
```sh
pnpm paperclipai configure --section database
```
Environment overrides:
- `PAPERCLIP_DB_BACKUP_ENABLED=true|false`
- `PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES=<minutes>`
- `PAPERCLIP_DB_BACKUP_RETENTION_DAYS=<days>`
- `PAPERCLIP_DB_BACKUP_DIR=/absolute/or/~/path`
## Secrets in Dev ## Secrets in Dev
Agent env vars now support secret references. By default, secret values are stored with local encryption and only secret refs are persisted in agent config. Agent env vars now support secret references. By default, secret values are stored with local encryption and only secret refs are persisted in agent config.

View File

@@ -0,0 +1,333 @@
import { existsSync, mkdirSync, readdirSync, statSync, unlinkSync } from "node:fs";
import { writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import postgres from "postgres";
export type RunDatabaseBackupOptions = {
connectionString: string;
backupDir: string;
retentionDays: number;
filenamePrefix?: string;
connectTimeoutSeconds?: number;
};
export type RunDatabaseBackupResult = {
backupFile: string;
sizeBytes: number;
prunedCount: number;
};
function timestamp(date: Date = new Date()): string {
const pad = (n: number) => String(n).padStart(2, "0");
return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}-${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`;
}
function pruneOldBackups(backupDir: string, retentionDays: number, filenamePrefix: string): number {
if (!existsSync(backupDir)) return 0;
const safeRetention = Math.max(1, Math.trunc(retentionDays));
const cutoff = Date.now() - safeRetention * 24 * 60 * 60 * 1000;
let pruned = 0;
for (const name of readdirSync(backupDir)) {
if (!name.startsWith(`${filenamePrefix}-`) || !name.endsWith(".sql")) continue;
const fullPath = resolve(backupDir, name);
const stat = statSync(fullPath);
if (stat.mtimeMs < cutoff) {
unlinkSync(fullPath);
pruned++;
}
}
return pruned;
}
function formatBackupSize(sizeBytes: number): string {
if (sizeBytes < 1024) return `${sizeBytes}B`;
if (sizeBytes < 1024 * 1024) return `${(sizeBytes / 1024).toFixed(1)}K`;
return `${(sizeBytes / (1024 * 1024)).toFixed(1)}M`;
}
export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise<RunDatabaseBackupResult> {
const filenamePrefix = opts.filenamePrefix ?? "paperclip";
const retentionDays = Math.max(1, Math.trunc(opts.retentionDays));
const connectTimeout = Math.max(1, Math.trunc(opts.connectTimeoutSeconds ?? 5));
const sql = postgres(opts.connectionString, { max: 1, connect_timeout: connectTimeout });
try {
await sql`SELECT 1`;
const lines: string[] = [];
const emit = (line: string) => lines.push(line);
emit("-- Paperclip database backup");
emit(`-- Created: ${new Date().toISOString()}`);
emit("");
emit("BEGIN;");
emit("");
// Get all enums
const enums = await sql<{ typname: string; labels: string[] }[]>`
SELECT t.typname, array_agg(e.enumlabel ORDER BY e.enumsortorder) AS labels
FROM pg_type t
JOIN pg_enum e ON t.oid = e.enumtypid
JOIN pg_namespace n ON t.typnamespace = n.oid
WHERE n.nspname = 'public'
GROUP BY t.typname
ORDER BY t.typname
`;
for (const e of enums) {
const labels = e.labels.map((l) => `'${l.replace(/'/g, "''")}'`).join(", ");
emit(`CREATE TYPE "public"."${e.typname}" AS ENUM (${labels});`);
}
if (enums.length > 0) emit("");
// Get tables in dependency order (referenced tables first)
const tables = await sql<{ tablename: string }[]>`
SELECT c.relname AS tablename
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'public'
AND c.relkind = 'r'
AND c.relname != '__drizzle_migrations'
ORDER BY c.relname
`;
// Get full CREATE TABLE DDL via column info
for (const { tablename } of tables) {
const columns = await sql<{
column_name: string;
data_type: string;
udt_name: string;
is_nullable: string;
column_default: string | null;
character_maximum_length: number | null;
numeric_precision: number | null;
numeric_scale: number | null;
}[]>`
SELECT column_name, data_type, udt_name, is_nullable, column_default,
character_maximum_length, numeric_precision, numeric_scale
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = ${tablename}
ORDER BY ordinal_position
`;
emit(`-- Table: ${tablename}`);
emit(`DROP TABLE IF EXISTS "${tablename}" CASCADE;`);
const colDefs: string[] = [];
for (const col of columns) {
let typeStr: string;
if (col.data_type === "USER-DEFINED") {
typeStr = `"${col.udt_name}"`;
} else if (col.data_type === "ARRAY") {
typeStr = `${col.udt_name.replace(/^_/, "")}[]`;
} else if (col.data_type === "character varying") {
typeStr = col.character_maximum_length
? `varchar(${col.character_maximum_length})`
: "varchar";
} else if (col.data_type === "numeric" && col.numeric_precision != null) {
typeStr =
col.numeric_scale != null
? `numeric(${col.numeric_precision}, ${col.numeric_scale})`
: `numeric(${col.numeric_precision})`;
} else {
typeStr = col.data_type;
}
let def = ` "${col.column_name}" ${typeStr}`;
if (col.column_default != null) def += ` DEFAULT ${col.column_default}`;
if (col.is_nullable === "NO") def += " NOT NULL";
colDefs.push(def);
}
// Primary key
const pk = await sql<{ constraint_name: string; column_names: string[] }[]>`
SELECT c.conname AS constraint_name,
array_agg(a.attname ORDER BY array_position(c.conkey, a.attnum)) AS column_names
FROM pg_constraint c
JOIN pg_class t ON t.oid = c.conrelid
JOIN pg_namespace n ON n.oid = t.relnamespace
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey)
WHERE n.nspname = 'public' AND t.relname = ${tablename} AND c.contype = 'p'
GROUP BY c.conname
`;
for (const p of pk) {
const cols = p.column_names.map((c) => `"${c}"`).join(", ");
colDefs.push(` CONSTRAINT "${p.constraint_name}" PRIMARY KEY (${cols})`);
}
emit(`CREATE TABLE "${tablename}" (`);
emit(colDefs.join(",\n"));
emit(");");
emit("");
}
// Foreign keys (after all tables created)
const fks = await sql<{
constraint_name: string;
source_table: string;
source_columns: string[];
target_table: string;
target_columns: string[];
update_rule: string;
delete_rule: string;
}[]>`
SELECT
c.conname AS constraint_name,
src.relname AS source_table,
array_agg(sa.attname ORDER BY array_position(c.conkey, sa.attnum)) AS source_columns,
tgt.relname AS target_table,
array_agg(ta.attname ORDER BY array_position(c.confkey, ta.attnum)) AS target_columns,
CASE c.confupdtype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS update_rule,
CASE c.confdeltype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS delete_rule
FROM pg_constraint c
JOIN pg_class src ON src.oid = c.conrelid
JOIN pg_class tgt ON tgt.oid = c.confrelid
JOIN pg_namespace n ON n.oid = src.relnamespace
JOIN pg_attribute sa ON sa.attrelid = src.oid AND sa.attnum = ANY(c.conkey)
JOIN pg_attribute ta ON ta.attrelid = tgt.oid AND ta.attnum = ANY(c.confkey)
WHERE c.contype = 'f' AND n.nspname = 'public'
GROUP BY c.conname, src.relname, tgt.relname, c.confupdtype, c.confdeltype
ORDER BY src.relname, c.conname
`;
if (fks.length > 0) {
emit("-- Foreign keys");
for (const fk of fks) {
const srcCols = fk.source_columns.map((c) => `"${c}"`).join(", ");
const tgtCols = fk.target_columns.map((c) => `"${c}"`).join(", ");
emit(
`ALTER TABLE "${fk.source_table}" ADD CONSTRAINT "${fk.constraint_name}" FOREIGN KEY (${srcCols}) REFERENCES "${fk.target_table}" (${tgtCols}) ON UPDATE ${fk.update_rule} ON DELETE ${fk.delete_rule};`,
);
}
emit("");
}
// Unique constraints
const uniques = await sql<{
constraint_name: string;
tablename: string;
column_names: string[];
}[]>`
SELECT c.conname AS constraint_name,
t.relname AS tablename,
array_agg(a.attname ORDER BY array_position(c.conkey, a.attnum)) AS column_names
FROM pg_constraint c
JOIN pg_class t ON t.oid = c.conrelid
JOIN pg_namespace n ON n.oid = t.relnamespace
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey)
WHERE n.nspname = 'public' AND c.contype = 'u'
GROUP BY c.conname, t.relname
ORDER BY t.relname, c.conname
`;
if (uniques.length > 0) {
emit("-- Unique constraints");
for (const u of uniques) {
const cols = u.column_names.map((c) => `"${c}"`).join(", ");
emit(`ALTER TABLE "${u.tablename}" ADD CONSTRAINT "${u.constraint_name}" UNIQUE (${cols});`);
}
emit("");
}
// Indexes (non-primary, non-unique-constraint)
const indexes = await sql<{ indexdef: string }[]>`
SELECT indexdef
FROM pg_indexes
WHERE schemaname = 'public'
AND indexname NOT IN (
SELECT conname FROM pg_constraint
WHERE connamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public')
)
ORDER BY tablename, indexname
`;
if (indexes.length > 0) {
emit("-- Indexes");
for (const idx of indexes) {
emit(`${idx.indexdef};`);
}
emit("");
}
// Dump data for each table
for (const { tablename } of tables) {
const count = await sql<{ n: number }[]>`
SELECT count(*)::int AS n FROM ${sql(tablename)}
`;
if ((count[0]?.n ?? 0) === 0) continue;
// Get column info for this table
const cols = await sql<{ column_name: string; data_type: string }[]>`
SELECT column_name, data_type
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = ${tablename}
ORDER BY ordinal_position
`;
const colNames = cols.map((c) => `"${c.column_name}"`).join(", ");
emit(`-- Data for: ${tablename} (${count[0]!.n} rows)`);
const rows = await sql`SELECT * FROM ${sql(tablename)}`.values();
for (const row of rows) {
const values = row.map((val: unknown) => {
if (val === null || val === undefined) return "NULL";
if (typeof val === "boolean") return val ? "true" : "false";
if (typeof val === "number") return String(val);
if (val instanceof Date) return `'${val.toISOString()}'`;
if (typeof val === "object") return `'${JSON.stringify(val).replace(/'/g, "''")}'`;
return `'${String(val).replace(/'/g, "''")}'`;
});
emit(`INSERT INTO "${tablename}" (${colNames}) VALUES (${values.join(", ")});`);
}
emit("");
}
// Sequence values
const sequences = await sql<{ sequence_name: string }[]>`
SELECT sequence_name
FROM information_schema.sequences
WHERE sequence_schema = 'public'
ORDER BY sequence_name
`;
if (sequences.length > 0) {
emit("-- Sequence values");
for (const seq of sequences) {
const val = await sql<{ last_value: string }[]>`
SELECT last_value::text FROM ${sql(seq.sequence_name)}
`;
if (val[0]) {
emit(`SELECT setval('"${seq.sequence_name}"', ${val[0].last_value});`);
}
}
emit("");
}
emit("COMMIT;");
emit("");
// Write the backup file
mkdirSync(opts.backupDir, { recursive: true });
const backupFile = resolve(opts.backupDir, `${filenamePrefix}-${timestamp()}.sql`);
await writeFile(backupFile, lines.join("\n"), "utf8");
const sizeBytes = statSync(backupFile).size;
const prunedCount = pruneOldBackups(opts.backupDir, retentionDays, filenamePrefix);
return {
backupFile,
sizeBytes,
prunedCount,
};
} finally {
await sql.end();
}
}
export function formatDatabaseBackupResult(result: RunDatabaseBackupResult): string {
const size = formatBackupSize(result.sizeBytes);
const pruned = result.prunedCount > 0 ? `; pruned ${result.prunedCount} old backup(s)` : "";
return `${result.backupFile} (${size}${pruned})`;
}

View File

@@ -1,338 +1,122 @@
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, unlinkSync } from "node:fs"; import { existsSync, readFileSync } from "node:fs";
import { writeFile } from "node:fs/promises"; import os from "node:os";
import { resolve } from "node:path"; import path from "node:path";
import postgres from "postgres"; import { formatDatabaseBackupResult, runDatabaseBackup } from "./backup-lib.js";
const PROJECT_ROOT = resolve(import.meta.dirname, "../../.."); type PartialConfig = {
const BACKUP_DIR = resolve(PROJECT_ROOT, "data/backups"); database?: {
const CONFIG_FILE = resolve(PROJECT_ROOT, ".paperclip/config.json"); mode?: "embedded-postgres" | "postgres";
const MAX_AGE_DAYS = 30; connectionString?: string;
embeddedPostgresPort?: number;
backup?: {
dir?: string;
retentionDays?: number;
};
};
};
function loadPort(): number { function expandHomePrefix(value: string): string {
try { if (value === "~") return os.homedir();
const raw = readFileSync(CONFIG_FILE, "utf8"); if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2));
const config = JSON.parse(raw); return value;
const port = config?.database?.embeddedPostgresPort;
if (typeof port === "number" && Number.isFinite(port)) return port;
} catch {}
return 54329;
} }
function timestamp(): string { function resolvePaperclipHomeDir(): string {
const d = new Date(); const envHome = process.env.PAPERCLIP_HOME?.trim();
const pad = (n: number) => String(n).padStart(2, "0"); if (envHome) return path.resolve(expandHomePrefix(envHome));
return `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}-${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}`; return path.resolve(os.homedir(), ".paperclip");
} }
function pruneOldBackups() { function resolvePaperclipInstanceId(): string {
if (!existsSync(BACKUP_DIR)) return; const raw = process.env.PAPERCLIP_INSTANCE_ID?.trim() || "default";
const cutoff = Date.now() - MAX_AGE_DAYS * 24 * 60 * 60 * 1000; if (!/^[a-zA-Z0-9_-]+$/.test(raw)) {
let pruned = 0; throw new Error(`Invalid PAPERCLIP_INSTANCE_ID '${raw}'.`);
for (const name of readdirSync(BACKUP_DIR)) {
if (!name.startsWith("paperclip-") || !name.endsWith(".sql")) continue;
const fullPath = resolve(BACKUP_DIR, name);
const stat = statSync(fullPath);
if (stat.mtimeMs < cutoff) {
unlinkSync(fullPath);
pruned++;
}
} }
if (pruned > 0) console.log(`Pruned ${pruned} backup(s) older than ${MAX_AGE_DAYS} days.`); return raw;
}
function resolveDefaultConfigPath(): string {
return path.resolve(resolvePaperclipHomeDir(), "instances", resolvePaperclipInstanceId(), "config.json");
}
function readConfig(configPath: string): PartialConfig | null {
if (!existsSync(configPath)) return null;
try {
const parsed = JSON.parse(readFileSync(configPath, "utf8"));
return typeof parsed === "object" && parsed ? (parsed as PartialConfig) : null;
} catch {
return null;
}
}
function asPositiveInt(value: unknown): number | null {
if (typeof value !== "number" || !Number.isFinite(value)) return null;
const rounded = Math.trunc(value);
return rounded > 0 ? rounded : null;
}
function resolveEmbeddedPort(config: PartialConfig | null): number {
return asPositiveInt(config?.database?.embeddedPostgresPort) ?? 54329;
}
function resolveConnectionString(config: PartialConfig | null): string {
const envUrl = process.env.DATABASE_URL?.trim();
if (envUrl) return envUrl;
if (config?.database?.mode === "postgres" && typeof config.database.connectionString === "string") {
const trimmed = config.database.connectionString.trim();
if (trimmed) return trimmed;
}
const port = resolveEmbeddedPort(config);
return `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
}
function resolveDefaultBackupDir(): string {
return path.resolve(resolvePaperclipHomeDir(), "instances", resolvePaperclipInstanceId(), "data", "backups");
}
function resolveBackupDir(config: PartialConfig | null): string {
const raw = config?.database?.backup?.dir;
if (typeof raw === "string" && raw.trim().length > 0) {
return path.resolve(expandHomePrefix(raw.trim()));
}
return resolveDefaultBackupDir();
}
function resolveRetentionDays(config: PartialConfig | null): number {
return asPositiveInt(config?.database?.backup?.retentionDays) ?? 30;
} }
async function main() { async function main() {
const port = loadPort(); const configPath = resolveDefaultConfigPath();
const connString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`; const config = readConfig(configPath);
const connectionString = resolveConnectionString(config);
const backupDir = resolveBackupDir(config);
const retentionDays = resolveRetentionDays(config);
console.log(`Connecting to embedded PostgreSQL on port ${port}...`); console.log(`Config path: ${configPath}`);
console.log(`Backing up database to: ${backupDir}`);
const sql = postgres(connString, { max: 1, connect_timeout: 5 }); console.log(`Retention window: ${retentionDays} day(s)`);
try { try {
// Verify connection const result = await runDatabaseBackup({
await sql`SELECT 1`; connectionString,
} catch (err: any) { backupDir,
console.error(`Error: Cannot connect to embedded PostgreSQL on port ${port}.`); retentionDays,
console.error(" Make sure the server is running (pnpm dev)."); filenamePrefix: "paperclip",
});
console.log(`Backup saved: ${formatDatabaseBackupResult(result)}`);
} catch (err) {
console.error("Backup failed.");
if (err instanceof Error) {
console.error(err.message);
} else {
console.error(String(err));
}
process.exit(1); process.exit(1);
} }
try {
const lines: string[] = [];
const emit = (line: string) => lines.push(line);
emit("-- Paperclip database backup");
emit(`-- Created: ${new Date().toISOString()}`);
emit(`-- Server port: ${port}`);
emit("");
emit("BEGIN;");
emit("");
// Get all enums
const enums = await sql<{ typname: string; labels: string[] }[]>`
SELECT t.typname, array_agg(e.enumlabel ORDER BY e.enumsortorder) AS labels
FROM pg_type t
JOIN pg_enum e ON t.oid = e.enumtypid
JOIN pg_namespace n ON t.typnamespace = n.oid
WHERE n.nspname = 'public'
GROUP BY t.typname
ORDER BY t.typname
`;
for (const e of enums) {
const labels = e.labels.map((l) => `'${l.replace(/'/g, "''")}'`).join(", ");
emit(`CREATE TYPE "public"."${e.typname}" AS ENUM (${labels});`);
}
if (enums.length > 0) emit("");
// Get tables in dependency order (referenced tables first)
const tables = await sql<{ tablename: string }[]>`
SELECT c.relname AS tablename
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'public'
AND c.relkind = 'r'
AND c.relname != '__drizzle_migrations'
ORDER BY c.relname
`;
// Get full CREATE TABLE DDL via column info
for (const { tablename } of tables) {
const columns = await sql<{
column_name: string;
data_type: string;
udt_name: string;
is_nullable: string;
column_default: string | null;
character_maximum_length: number | null;
numeric_precision: number | null;
numeric_scale: number | null;
}[]>`
SELECT column_name, data_type, udt_name, is_nullable, column_default,
character_maximum_length, numeric_precision, numeric_scale
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = ${tablename}
ORDER BY ordinal_position
`;
emit(`-- Table: ${tablename}`);
emit(`DROP TABLE IF EXISTS "${tablename}" CASCADE;`);
const colDefs: string[] = [];
for (const col of columns) {
let typeStr: string;
if (col.data_type === "USER-DEFINED") {
typeStr = `"${col.udt_name}"`;
} else if (col.data_type === "ARRAY") {
typeStr = `${col.udt_name.replace(/^_/, "")}[]`;
} else if (col.data_type === "character varying") {
typeStr = col.character_maximum_length
? `varchar(${col.character_maximum_length})`
: "varchar";
} else if (col.data_type === "numeric" && col.numeric_precision != null) {
typeStr =
col.numeric_scale != null
? `numeric(${col.numeric_precision}, ${col.numeric_scale})`
: `numeric(${col.numeric_precision})`;
} else {
typeStr = col.data_type;
}
let def = ` "${col.column_name}" ${typeStr}`;
if (col.column_default != null) def += ` DEFAULT ${col.column_default}`;
if (col.is_nullable === "NO") def += " NOT NULL";
colDefs.push(def);
}
// Primary key
const pk = await sql<{ constraint_name: string; column_names: string[] }[]>`
SELECT c.conname AS constraint_name,
array_agg(a.attname ORDER BY array_position(c.conkey, a.attnum)) AS column_names
FROM pg_constraint c
JOIN pg_class t ON t.oid = c.conrelid
JOIN pg_namespace n ON n.oid = t.relnamespace
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey)
WHERE n.nspname = 'public' AND t.relname = ${tablename} AND c.contype = 'p'
GROUP BY c.conname
`;
for (const p of pk) {
const cols = p.column_names.map((c) => `"${c}"`).join(", ");
colDefs.push(` CONSTRAINT "${p.constraint_name}" PRIMARY KEY (${cols})`);
}
emit(`CREATE TABLE "${tablename}" (`);
emit(colDefs.join(",\n"));
emit(");");
emit("");
}
// Foreign keys (after all tables created)
const fks = await sql<{
constraint_name: string;
source_table: string;
source_columns: string[];
target_table: string;
target_columns: string[];
update_rule: string;
delete_rule: string;
}[]>`
SELECT
c.conname AS constraint_name,
src.relname AS source_table,
array_agg(sa.attname ORDER BY array_position(c.conkey, sa.attnum)) AS source_columns,
tgt.relname AS target_table,
array_agg(ta.attname ORDER BY array_position(c.confkey, ta.attnum)) AS target_columns,
CASE c.confupdtype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS update_rule,
CASE c.confdeltype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS delete_rule
FROM pg_constraint c
JOIN pg_class src ON src.oid = c.conrelid
JOIN pg_class tgt ON tgt.oid = c.confrelid
JOIN pg_namespace n ON n.oid = src.relnamespace
JOIN pg_attribute sa ON sa.attrelid = src.oid AND sa.attnum = ANY(c.conkey)
JOIN pg_attribute ta ON ta.attrelid = tgt.oid AND ta.attnum = ANY(c.confkey)
WHERE c.contype = 'f' AND n.nspname = 'public'
GROUP BY c.conname, src.relname, tgt.relname, c.confupdtype, c.confdeltype
ORDER BY src.relname, c.conname
`;
if (fks.length > 0) {
emit("-- Foreign keys");
for (const fk of fks) {
const srcCols = fk.source_columns.map((c) => `"${c}"`).join(", ");
const tgtCols = fk.target_columns.map((c) => `"${c}"`).join(", ");
emit(
`ALTER TABLE "${fk.source_table}" ADD CONSTRAINT "${fk.constraint_name}" FOREIGN KEY (${srcCols}) REFERENCES "${fk.target_table}" (${tgtCols}) ON UPDATE ${fk.update_rule} ON DELETE ${fk.delete_rule};`,
);
}
emit("");
}
// Unique constraints
const uniques = await sql<{
constraint_name: string;
tablename: string;
column_names: string[];
}[]>`
SELECT c.conname AS constraint_name,
t.relname AS tablename,
array_agg(a.attname ORDER BY array_position(c.conkey, a.attnum)) AS column_names
FROM pg_constraint c
JOIN pg_class t ON t.oid = c.conrelid
JOIN pg_namespace n ON n.oid = t.relnamespace
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey)
WHERE n.nspname = 'public' AND c.contype = 'u'
GROUP BY c.conname, t.relname
ORDER BY t.relname, c.conname
`;
if (uniques.length > 0) {
emit("-- Unique constraints");
for (const u of uniques) {
const cols = u.column_names.map((c) => `"${c}"`).join(", ");
emit(`ALTER TABLE "${u.tablename}" ADD CONSTRAINT "${u.constraint_name}" UNIQUE (${cols});`);
}
emit("");
}
// Indexes (non-primary, non-unique-constraint)
const indexes = await sql<{ indexdef: string }[]>`
SELECT indexdef
FROM pg_indexes
WHERE schemaname = 'public'
AND indexname NOT IN (
SELECT conname FROM pg_constraint
WHERE connamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public')
)
ORDER BY tablename, indexname
`;
if (indexes.length > 0) {
emit("-- Indexes");
for (const idx of indexes) {
emit(`${idx.indexdef};`);
}
emit("");
}
// Dump data for each table
for (const { tablename } of tables) {
const count = await sql<{ n: number }[]>`
SELECT count(*)::int AS n FROM ${sql(tablename)}
`;
if ((count[0]?.n ?? 0) === 0) continue;
// Get column info for this table
const cols = await sql<{ column_name: string; data_type: string }[]>`
SELECT column_name, data_type
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = ${tablename}
ORDER BY ordinal_position
`;
const colNames = cols.map((c) => `"${c.column_name}"`).join(", ");
emit(`-- Data for: ${tablename} (${count[0]!.n} rows)`);
const rows = await sql`SELECT * FROM ${sql(tablename)}`.values();
for (const row of rows) {
const values = row.map((val: any) => {
if (val === null || val === undefined) return "NULL";
if (typeof val === "boolean") return val ? "true" : "false";
if (typeof val === "number") return String(val);
if (val instanceof Date) return `'${val.toISOString()}'`;
if (typeof val === "object") return `'${JSON.stringify(val).replace(/'/g, "''")}'`;
return `'${String(val).replace(/'/g, "''")}'`;
});
emit(`INSERT INTO "${tablename}" (${colNames}) VALUES (${values.join(", ")});`);
}
emit("");
}
// Sequence values
const sequences = await sql<{ sequence_name: string }[]>`
SELECT sequence_name
FROM information_schema.sequences
WHERE sequence_schema = 'public'
ORDER BY sequence_name
`;
if (sequences.length > 0) {
emit("-- Sequence values");
for (const seq of sequences) {
const val = await sql<{ last_value: string }[]>`
SELECT last_value::text FROM ${sql(seq.sequence_name)}
`;
if (val[0]) {
emit(`SELECT setval('"${seq.sequence_name}"', ${val[0].last_value});`);
}
}
emit("");
}
emit("COMMIT;");
emit("");
// Write the backup file
mkdirSync(BACKUP_DIR, { recursive: true });
const backupFile = resolve(BACKUP_DIR, `paperclip-${timestamp()}.sql`);
await writeFile(backupFile, lines.join("\n"), "utf8");
const sizeBytes = statSync(backupFile).size;
const sizeStr =
sizeBytes < 1024
? `${sizeBytes}B`
: sizeBytes < 1024 * 1024
? `${(sizeBytes / 1024).toFixed(1)}K`
: `${(sizeBytes / (1024 * 1024)).toFixed(1)}M`;
console.log(`Backup saved: ${backupFile} (${sizeStr})`);
pruneOldBackups();
} finally {
await sql.end();
}
} }
main().catch((err) => { await main();
console.error(err);
process.exit(1);
});

View File

@@ -10,4 +10,10 @@ export {
type MigrationBootstrapResult, type MigrationBootstrapResult,
type Db, type Db,
} from "./client.js"; } from "./client.js";
export {
runDatabaseBackup,
formatDatabaseBackupResult,
type RunDatabaseBackupOptions,
type RunDatabaseBackupResult,
} from "./backup-lib.js";
export * from "./schema/index.js"; export * from "./schema/index.js";

View File

@@ -18,11 +18,24 @@ export const llmConfigSchema = z.object({
apiKey: z.string().optional(), apiKey: z.string().optional(),
}); });
export const databaseBackupConfigSchema = z.object({
enabled: z.boolean().default(true),
intervalMinutes: z.number().int().min(1).max(7 * 24 * 60).default(60),
retentionDays: z.number().int().min(1).max(3650).default(30),
dir: z.string().default("~/.paperclip/instances/default/data/backups"),
});
export const databaseConfigSchema = z.object({ export const databaseConfigSchema = z.object({
mode: z.enum(["embedded-postgres", "postgres"]).default("embedded-postgres"), mode: z.enum(["embedded-postgres", "postgres"]).default("embedded-postgres"),
connectionString: z.string().optional(), connectionString: z.string().optional(),
embeddedPostgresDataDir: z.string().default("~/.paperclip/instances/default/db"), embeddedPostgresDataDir: z.string().default("~/.paperclip/instances/default/db"),
embeddedPostgresPort: z.number().int().min(1).max(65535).default(54329), embeddedPostgresPort: z.number().int().min(1).max(65535).default(54329),
backup: databaseBackupConfigSchema.default({
enabled: true,
intervalMinutes: 60,
retentionDays: 30,
dir: "~/.paperclip/instances/default/data/backups",
}),
}); });
export const loggingConfigSchema = z.object({ export const loggingConfigSchema = z.object({
@@ -160,3 +173,4 @@ export type SecretsConfig = z.infer<typeof secretsConfigSchema>;
export type SecretsLocalEncryptedConfig = z.infer<typeof secretsLocalEncryptedConfigSchema>; export type SecretsLocalEncryptedConfig = z.infer<typeof secretsLocalEncryptedConfigSchema>;
export type AuthConfig = z.infer<typeof authConfigSchema>; export type AuthConfig = z.infer<typeof authConfigSchema>;
export type ConfigMeta = z.infer<typeof configMetaSchema>; export type ConfigMeta = z.infer<typeof configMetaSchema>;
export type DatabaseBackupConfig = z.infer<typeof databaseBackupConfigSchema>;

View File

@@ -243,6 +243,7 @@ export {
paperclipConfigSchema, paperclipConfigSchema,
configMetaSchema, configMetaSchema,
llmConfigSchema, llmConfigSchema,
databaseBackupConfigSchema,
databaseConfigSchema, databaseConfigSchema,
loggingConfigSchema, loggingConfigSchema,
serverConfigSchema, serverConfigSchema,
@@ -254,6 +255,7 @@ export {
secretsLocalEncryptedConfigSchema, secretsLocalEncryptedConfigSchema,
type PaperclipConfig, type PaperclipConfig,
type LlmConfig, type LlmConfig,
type DatabaseBackupConfig,
type DatabaseConfig, type DatabaseConfig,
type LoggingConfig, type LoggingConfig,
type ServerConfig, type ServerConfig,

View File

@@ -1,7 +1,8 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail set -euo pipefail
# Backup the embedded PostgreSQL database to data/backups/ # Backup the configured Paperclip database to the configured backup directory
# (default: ~/.paperclip/instances/<instance-id>/data/backups)
# #
# Usage: # Usage:
# ./scripts/backup-db.sh # ./scripts/backup-db.sh

View File

@@ -15,6 +15,7 @@ import {
type StorageProvider, type StorageProvider,
} from "@paperclipai/shared"; } from "@paperclipai/shared";
import { import {
resolveDefaultBackupDir,
resolveDefaultEmbeddedPostgresDir, resolveDefaultEmbeddedPostgresDir,
resolveDefaultSecretsKeyFilePath, resolveDefaultSecretsKeyFilePath,
resolveDefaultStorageDir, resolveDefaultStorageDir,
@@ -40,6 +41,10 @@ export interface Config {
databaseUrl: string | undefined; databaseUrl: string | undefined;
embeddedPostgresDataDir: string; embeddedPostgresDataDir: string;
embeddedPostgresPort: number; embeddedPostgresPort: number;
databaseBackupEnabled: boolean;
databaseBackupIntervalMinutes: number;
databaseBackupRetentionDays: number;
databaseBackupDir: string;
serveUi: boolean; serveUi: boolean;
uiDevMiddleware: boolean; uiDevMiddleware: boolean;
secretsProvider: SecretProvider; secretsProvider: SecretProvider;
@@ -66,6 +71,7 @@ export function loadConfig(): Config {
fileDatabaseMode === "postgres" fileDatabaseMode === "postgres"
? fileConfig?.database.connectionString ? fileConfig?.database.connectionString
: undefined; : undefined;
const fileDatabaseBackup = fileConfig?.database.backup;
const fileSecrets = fileConfig?.secrets; const fileSecrets = fileConfig?.secrets;
const fileStorage = fileConfig?.storage; const fileStorage = fileConfig?.storage;
const strictModeFromEnv = process.env.PAPERCLIP_SECRETS_STRICT_MODE; const strictModeFromEnv = process.env.PAPERCLIP_SECRETS_STRICT_MODE;
@@ -148,6 +154,27 @@ export function loadConfig(): Config {
companyDeletionEnvRaw !== undefined companyDeletionEnvRaw !== undefined
? companyDeletionEnvRaw === "true" ? companyDeletionEnvRaw === "true"
: deploymentMode === "local_trusted"; : deploymentMode === "local_trusted";
const databaseBackupEnabled =
process.env.PAPERCLIP_DB_BACKUP_ENABLED !== undefined
? process.env.PAPERCLIP_DB_BACKUP_ENABLED === "true"
: (fileDatabaseBackup?.enabled ?? true);
const databaseBackupIntervalMinutes = Math.max(
1,
Number(process.env.PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES) ||
fileDatabaseBackup?.intervalMinutes ||
60,
);
const databaseBackupRetentionDays = Math.max(
1,
Number(process.env.PAPERCLIP_DB_BACKUP_RETENTION_DAYS) ||
fileDatabaseBackup?.retentionDays ||
30,
);
const databaseBackupDir = resolveHomeAwarePath(
process.env.PAPERCLIP_DB_BACKUP_DIR ??
fileDatabaseBackup?.dir ??
resolveDefaultBackupDir(),
);
return { return {
deploymentMode, deploymentMode,
@@ -163,6 +190,10 @@ export function loadConfig(): Config {
fileConfig?.database.embeddedPostgresDataDir ?? resolveDefaultEmbeddedPostgresDir(), fileConfig?.database.embeddedPostgresDataDir ?? resolveDefaultEmbeddedPostgresDir(),
), ),
embeddedPostgresPort: fileConfig?.database.embeddedPostgresPort ?? 54329, embeddedPostgresPort: fileConfig?.database.embeddedPostgresPort ?? 54329,
databaseBackupEnabled,
databaseBackupIntervalMinutes,
databaseBackupRetentionDays,
databaseBackupDir,
serveUi: serveUi:
process.env.SERVE_UI !== undefined process.env.SERVE_UI !== undefined
? process.env.SERVE_UI === "true" ? process.env.SERVE_UI === "true"

View File

@@ -49,6 +49,10 @@ export function resolveDefaultStorageDir(): string {
return path.resolve(resolvePaperclipInstanceRoot(), "data", "storage"); return path.resolve(resolvePaperclipInstanceRoot(), "data", "storage");
} }
export function resolveDefaultBackupDir(): string {
return path.resolve(resolvePaperclipInstanceRoot(), "data", "backups");
}
export function resolveDefaultAgentWorkspaceDir(agentId: string): string { export function resolveDefaultAgentWorkspaceDir(agentId: string): string {
const trimmed = agentId.trim(); const trimmed = agentId.trim();
if (!PATH_SEGMENT_RE.test(trimmed)) { if (!PATH_SEGMENT_RE.test(trimmed)) {

View File

@@ -12,6 +12,8 @@ import {
inspectMigrations, inspectMigrations,
applyPendingMigrations, applyPendingMigrations,
reconcilePendingMigrationHistory, reconcilePendingMigrationHistory,
formatDatabaseBackupResult,
runDatabaseBackup,
authUsers, authUsers,
companies, companies,
companyMemberships, companyMemberships,
@@ -220,6 +222,7 @@ let db;
let embeddedPostgres: EmbeddedPostgresInstance | null = null; let embeddedPostgres: EmbeddedPostgresInstance | null = null;
let embeddedPostgresStartedByThisProcess = false; let embeddedPostgresStartedByThisProcess = false;
let migrationSummary: MigrationSummary = "skipped"; let migrationSummary: MigrationSummary = "skipped";
let activeDatabaseConnectionString: string;
let startupDbInfo: let startupDbInfo:
| { mode: "external-postgres"; connectionString: string } | { mode: "external-postgres"; connectionString: string }
| { mode: "embedded-postgres"; dataDir: string; port: number }; | { mode: "embedded-postgres"; dataDir: string; port: number };
@@ -228,6 +231,7 @@ if (config.databaseUrl) {
db = createDb(config.databaseUrl); db = createDb(config.databaseUrl);
logger.info("Using external PostgreSQL via DATABASE_URL/config"); logger.info("Using external PostgreSQL via DATABASE_URL/config");
activeDatabaseConnectionString = config.databaseUrl;
startupDbInfo = { mode: "external-postgres", connectionString: config.databaseUrl }; startupDbInfo = { mode: "external-postgres", connectionString: config.databaseUrl };
} else { } else {
const moduleName = "embedded-postgres"; const moduleName = "embedded-postgres";
@@ -364,6 +368,7 @@ if (config.databaseUrl) {
db = createDb(embeddedConnectionString); db = createDb(embeddedConnectionString);
logger.info("Embedded PostgreSQL ready"); logger.info("Embedded PostgreSQL ready");
activeDatabaseConnectionString = embeddedConnectionString;
startupDbInfo = { mode: "embedded-postgres", dataDir, port }; startupDbInfo = { mode: "embedded-postgres", dataDir, port };
} }
@@ -489,6 +494,54 @@ if (config.heartbeatSchedulerEnabled) {
}, config.heartbeatSchedulerIntervalMs); }, config.heartbeatSchedulerIntervalMs);
} }
if (config.databaseBackupEnabled) {
const backupIntervalMs = config.databaseBackupIntervalMinutes * 60 * 1000;
let backupInFlight = false;
const runScheduledBackup = async () => {
if (backupInFlight) {
logger.warn("Skipping scheduled database backup because a previous backup is still running");
return;
}
backupInFlight = true;
try {
const result = await runDatabaseBackup({
connectionString: activeDatabaseConnectionString,
backupDir: config.databaseBackupDir,
retentionDays: config.databaseBackupRetentionDays,
filenamePrefix: "paperclip",
});
logger.info(
{
backupFile: result.backupFile,
sizeBytes: result.sizeBytes,
prunedCount: result.prunedCount,
backupDir: config.databaseBackupDir,
retentionDays: config.databaseBackupRetentionDays,
},
`Automatic database backup complete: ${formatDatabaseBackupResult(result)}`,
);
} catch (err) {
logger.error({ err, backupDir: config.databaseBackupDir }, "Automatic database backup failed");
} finally {
backupInFlight = false;
}
};
logger.info(
{
intervalMinutes: config.databaseBackupIntervalMinutes,
retentionDays: config.databaseBackupRetentionDays,
backupDir: config.databaseBackupDir,
},
"Automatic database backups enabled",
);
setInterval(() => {
void runScheduledBackup();
}, backupIntervalMs);
}
server.listen(listenPort, config.host, () => { server.listen(listenPort, config.host, () => {
logger.info(`Server listening on ${config.host}:${listenPort}`); logger.info(`Server listening on ${config.host}:${listenPort}`);
if (process.env.PAPERCLIP_OPEN_ON_LISTEN === "true") { if (process.env.PAPERCLIP_OPEN_ON_LISTEN === "true") {
@@ -515,6 +568,10 @@ server.listen(listenPort, config.host, () => {
migrationSummary, migrationSummary,
heartbeatSchedulerEnabled: config.heartbeatSchedulerEnabled, heartbeatSchedulerEnabled: config.heartbeatSchedulerEnabled,
heartbeatSchedulerIntervalMs: config.heartbeatSchedulerIntervalMs, heartbeatSchedulerIntervalMs: config.heartbeatSchedulerIntervalMs,
databaseBackupEnabled: config.databaseBackupEnabled,
databaseBackupIntervalMinutes: config.databaseBackupIntervalMinutes,
databaseBackupRetentionDays: config.databaseBackupRetentionDays,
databaseBackupDir: config.databaseBackupDir,
}); });
const boardClaimUrl = getBoardClaimWarningUrl(config.host, listenPort); const boardClaimUrl = getBoardClaimWarningUrl(config.host, listenPort);

View File

@@ -29,6 +29,10 @@ type StartupBannerOptions = {
migrationSummary: string; migrationSummary: string;
heartbeatSchedulerEnabled: boolean; heartbeatSchedulerEnabled: boolean;
heartbeatSchedulerIntervalMs: number; heartbeatSchedulerIntervalMs: number;
databaseBackupEnabled: boolean;
databaseBackupIntervalMinutes: number;
databaseBackupRetentionDays: number;
databaseBackupDir: string;
}; };
const ansi = { const ansi = {
@@ -125,6 +129,9 @@ export function printStartupBanner(opts: StartupBannerOptions): void {
const heartbeat = opts.heartbeatSchedulerEnabled const heartbeat = opts.heartbeatSchedulerEnabled
? `enabled ${color(`(${opts.heartbeatSchedulerIntervalMs}ms)`, "dim")}` ? `enabled ${color(`(${opts.heartbeatSchedulerIntervalMs}ms)`, "dim")}`
: color("disabled", "yellow"); : color("disabled", "yellow");
const dbBackup = opts.databaseBackupEnabled
? `enabled ${color(`(every ${opts.databaseBackupIntervalMinutes}m, keep ${opts.databaseBackupRetentionDays}d)`, "dim")}`
: color("disabled", "yellow");
const art = [ const art = [
color("██████╗ █████╗ ██████╗ ███████╗██████╗ ██████╗██╗ ██╗██████╗ ", "cyan"), color("██████╗ █████╗ ██████╗ ███████╗██████╗ ██████╗██╗ ██╗██████╗ ", "cyan"),
@@ -154,6 +161,8 @@ export function printStartupBanner(opts: StartupBannerOptions): void {
: color(agentJwtSecret.message, "yellow"), : color(agentJwtSecret.message, "yellow"),
), ),
row("Heartbeat", heartbeat), row("Heartbeat", heartbeat),
row("DB Backup", dbBackup),
row("Backup Dir", opts.databaseBackupDir),
row("Config", configPath), row("Config", configPath),
agentJwtSecret.status === "warn" agentJwtSecret.status === "warn"
? color(" ───────────────────────────────────────────────────────", "yellow") ? color(" ───────────────────────────────────────────────────────", "yellow")