Add configurable automatic database backup scheduling
This commit is contained in:
@@ -21,6 +21,12 @@ function writeBaseConfig(configPath: string) {
|
|||||||
mode: "embedded-postgres",
|
mode: "embedded-postgres",
|
||||||
embeddedPostgresDataDir: "/tmp/paperclip-db",
|
embeddedPostgresDataDir: "/tmp/paperclip-db",
|
||||||
embeddedPostgresPort: 54329,
|
embeddedPostgresPort: 54329,
|
||||||
|
backup: {
|
||||||
|
enabled: true,
|
||||||
|
intervalMinutes: 60,
|
||||||
|
retentionDays: 30,
|
||||||
|
dir: "/tmp/paperclip-backups",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
logging: {
|
logging: {
|
||||||
mode: "file",
|
mode: "file",
|
||||||
@@ -68,4 +74,3 @@ describe("allowed-hostname command", () => {
|
|||||||
expect(raw.server.allowedHostnames).toEqual(["dotta-macbook-pro"]);
|
expect(raw.server.allowedHostnames).toEqual(["dotta-macbook-pro"]);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import { defaultSecretsConfig, promptSecrets } from "../prompts/secrets.js";
|
|||||||
import { defaultStorageConfig, promptStorage } from "../prompts/storage.js";
|
import { defaultStorageConfig, promptStorage } from "../prompts/storage.js";
|
||||||
import { promptServer } from "../prompts/server.js";
|
import { promptServer } from "../prompts/server.js";
|
||||||
import {
|
import {
|
||||||
|
resolveDefaultBackupDir,
|
||||||
resolveDefaultEmbeddedPostgresDir,
|
resolveDefaultEmbeddedPostgresDir,
|
||||||
resolveDefaultLogsDir,
|
resolveDefaultLogsDir,
|
||||||
resolvePaperclipInstanceId,
|
resolvePaperclipInstanceId,
|
||||||
@@ -39,6 +40,12 @@ function defaultConfig(): PaperclipConfig {
|
|||||||
mode: "embedded-postgres",
|
mode: "embedded-postgres",
|
||||||
embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(instanceId),
|
embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(instanceId),
|
||||||
embeddedPostgresPort: 54329,
|
embeddedPostgresPort: 54329,
|
||||||
|
backup: {
|
||||||
|
enabled: true,
|
||||||
|
intervalMinutes: 60,
|
||||||
|
retentionDays: 30,
|
||||||
|
dir: resolveDefaultBackupDir(instanceId),
|
||||||
|
},
|
||||||
},
|
},
|
||||||
logging: {
|
logging: {
|
||||||
mode: "file",
|
mode: "file",
|
||||||
@@ -118,7 +125,7 @@ export async function configure(opts: {
|
|||||||
|
|
||||||
switch (section) {
|
switch (section) {
|
||||||
case "database":
|
case "database":
|
||||||
config.database = await promptDatabase();
|
config.database = await promptDatabase(config.database);
|
||||||
break;
|
break;
|
||||||
case "llm": {
|
case "llm": {
|
||||||
const llm = await promptLlm();
|
const llm = await promptLlm();
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import { defaultStorageConfig, promptStorage } from "../prompts/storage.js";
|
|||||||
import { promptServer } from "../prompts/server.js";
|
import { promptServer } from "../prompts/server.js";
|
||||||
import {
|
import {
|
||||||
describeLocalInstancePaths,
|
describeLocalInstancePaths,
|
||||||
|
resolveDefaultBackupDir,
|
||||||
resolveDefaultEmbeddedPostgresDir,
|
resolveDefaultEmbeddedPostgresDir,
|
||||||
resolveDefaultLogsDir,
|
resolveDefaultLogsDir,
|
||||||
resolvePaperclipInstanceId,
|
resolvePaperclipInstanceId,
|
||||||
@@ -35,6 +36,12 @@ function quickstartDefaults(): Pick<PaperclipConfig, "database" | "logging" | "s
|
|||||||
mode: "embedded-postgres",
|
mode: "embedded-postgres",
|
||||||
embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(instanceId),
|
embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(instanceId),
|
||||||
embeddedPostgresPort: 54329,
|
embeddedPostgresPort: 54329,
|
||||||
|
backup: {
|
||||||
|
enabled: true,
|
||||||
|
intervalMinutes: 60,
|
||||||
|
retentionDays: 30,
|
||||||
|
dir: resolveDefaultBackupDir(instanceId),
|
||||||
|
},
|
||||||
},
|
},
|
||||||
logging: {
|
logging: {
|
||||||
mode: "file",
|
mode: "file",
|
||||||
@@ -120,7 +127,7 @@ export async function onboard(opts: OnboardOptions): Promise<void> {
|
|||||||
|
|
||||||
if (setupMode === "advanced") {
|
if (setupMode === "advanced") {
|
||||||
p.log.step(pc.bold("Database"));
|
p.log.step(pc.bold("Database"));
|
||||||
database = await promptDatabase();
|
database = await promptDatabase(database);
|
||||||
|
|
||||||
if (database.mode === "postgres" && database.connectionString) {
|
if (database.mode === "postgres" && database.connectionString) {
|
||||||
const s = p.spinner();
|
const s = p.spinner();
|
||||||
|
|||||||
@@ -49,6 +49,10 @@ export function resolveDefaultStorageDir(instanceId?: string): string {
|
|||||||
return path.resolve(resolvePaperclipInstanceRoot(instanceId), "data", "storage");
|
return path.resolve(resolvePaperclipInstanceRoot(instanceId), "data", "storage");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function resolveDefaultBackupDir(instanceId?: string): string {
|
||||||
|
return path.resolve(resolvePaperclipInstanceRoot(instanceId), "data", "backups");
|
||||||
|
}
|
||||||
|
|
||||||
export function expandHomePrefix(value: string): string {
|
export function expandHomePrefix(value: string): string {
|
||||||
if (value === "~") return os.homedir();
|
if (value === "~") return os.homedir();
|
||||||
if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2));
|
if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2));
|
||||||
@@ -64,6 +68,7 @@ export function describeLocalInstancePaths(instanceId?: string) {
|
|||||||
instanceRoot,
|
instanceRoot,
|
||||||
configPath: resolveDefaultConfigPath(resolvedInstanceId),
|
configPath: resolveDefaultConfigPath(resolvedInstanceId),
|
||||||
embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(resolvedInstanceId),
|
embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(resolvedInstanceId),
|
||||||
|
backupDir: resolveDefaultBackupDir(resolvedInstanceId),
|
||||||
logDir: resolveDefaultLogsDir(resolvedInstanceId),
|
logDir: resolveDefaultLogsDir(resolvedInstanceId),
|
||||||
secretsKeyFilePath: resolveDefaultSecretsKeyFilePath(resolvedInstanceId),
|
secretsKeyFilePath: resolveDefaultSecretsKeyFilePath(resolvedInstanceId),
|
||||||
storageDir: resolveDefaultStorageDir(resolvedInstanceId),
|
storageDir: resolveDefaultStorageDir(resolvedInstanceId),
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ export {
|
|||||||
paperclipConfigSchema,
|
paperclipConfigSchema,
|
||||||
configMetaSchema,
|
configMetaSchema,
|
||||||
llmConfigSchema,
|
llmConfigSchema,
|
||||||
|
databaseBackupConfigSchema,
|
||||||
databaseConfigSchema,
|
databaseConfigSchema,
|
||||||
loggingConfigSchema,
|
loggingConfigSchema,
|
||||||
serverConfigSchema,
|
serverConfigSchema,
|
||||||
@@ -13,6 +14,7 @@ export {
|
|||||||
secretsLocalEncryptedConfigSchema,
|
secretsLocalEncryptedConfigSchema,
|
||||||
type PaperclipConfig,
|
type PaperclipConfig,
|
||||||
type LlmConfig,
|
type LlmConfig,
|
||||||
|
type DatabaseBackupConfig,
|
||||||
type DatabaseConfig,
|
type DatabaseConfig,
|
||||||
type LoggingConfig,
|
type LoggingConfig,
|
||||||
type ServerConfig,
|
type ServerConfig,
|
||||||
|
|||||||
@@ -1,9 +1,26 @@
|
|||||||
import * as p from "@clack/prompts";
|
import * as p from "@clack/prompts";
|
||||||
import type { DatabaseConfig } from "../config/schema.js";
|
import type { DatabaseConfig } from "../config/schema.js";
|
||||||
import { resolveDefaultEmbeddedPostgresDir, resolvePaperclipInstanceId } from "../config/home.js";
|
import {
|
||||||
|
resolveDefaultBackupDir,
|
||||||
|
resolveDefaultEmbeddedPostgresDir,
|
||||||
|
resolvePaperclipInstanceId,
|
||||||
|
} from "../config/home.js";
|
||||||
|
|
||||||
export async function promptDatabase(): Promise<DatabaseConfig> {
|
export async function promptDatabase(current?: DatabaseConfig): Promise<DatabaseConfig> {
|
||||||
const defaultEmbeddedDir = resolveDefaultEmbeddedPostgresDir(resolvePaperclipInstanceId());
|
const instanceId = resolvePaperclipInstanceId();
|
||||||
|
const defaultEmbeddedDir = resolveDefaultEmbeddedPostgresDir(instanceId);
|
||||||
|
const defaultBackupDir = resolveDefaultBackupDir(instanceId);
|
||||||
|
const base: DatabaseConfig = current ?? {
|
||||||
|
mode: "embedded-postgres",
|
||||||
|
embeddedPostgresDataDir: defaultEmbeddedDir,
|
||||||
|
embeddedPostgresPort: 54329,
|
||||||
|
backup: {
|
||||||
|
enabled: true,
|
||||||
|
intervalMinutes: 60,
|
||||||
|
retentionDays: 30,
|
||||||
|
dir: defaultBackupDir,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
const mode = await p.select({
|
const mode = await p.select({
|
||||||
message: "Database mode",
|
message: "Database mode",
|
||||||
@@ -11,6 +28,7 @@ export async function promptDatabase(): Promise<DatabaseConfig> {
|
|||||||
{ value: "embedded-postgres" as const, label: "Embedded PostgreSQL (managed locally)", hint: "recommended" },
|
{ value: "embedded-postgres" as const, label: "Embedded PostgreSQL (managed locally)", hint: "recommended" },
|
||||||
{ value: "postgres" as const, label: "PostgreSQL (external server)" },
|
{ value: "postgres" as const, label: "PostgreSQL (external server)" },
|
||||||
],
|
],
|
||||||
|
initialValue: base.mode,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (p.isCancel(mode)) {
|
if (p.isCancel(mode)) {
|
||||||
@@ -18,9 +36,14 @@ export async function promptDatabase(): Promise<DatabaseConfig> {
|
|||||||
process.exit(0);
|
process.exit(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let connectionString: string | undefined = base.connectionString;
|
||||||
|
let embeddedPostgresDataDir = base.embeddedPostgresDataDir || defaultEmbeddedDir;
|
||||||
|
let embeddedPostgresPort = base.embeddedPostgresPort || 54329;
|
||||||
|
|
||||||
if (mode === "postgres") {
|
if (mode === "postgres") {
|
||||||
const connectionString = await p.text({
|
const value = await p.text({
|
||||||
message: "PostgreSQL connection string",
|
message: "PostgreSQL connection string",
|
||||||
|
defaultValue: base.connectionString ?? "",
|
||||||
placeholder: "postgres://user:pass@localhost:5432/paperclip",
|
placeholder: "postgres://user:pass@localhost:5432/paperclip",
|
||||||
validate: (val) => {
|
validate: (val) => {
|
||||||
if (!val) return "Connection string is required for PostgreSQL mode";
|
if (!val) return "Connection string is required for PostgreSQL mode";
|
||||||
@@ -28,33 +51,29 @@ export async function promptDatabase(): Promise<DatabaseConfig> {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
if (p.isCancel(connectionString)) {
|
if (p.isCancel(value)) {
|
||||||
p.cancel("Setup cancelled.");
|
p.cancel("Setup cancelled.");
|
||||||
process.exit(0);
|
process.exit(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
connectionString = value;
|
||||||
mode: "postgres",
|
} else {
|
||||||
connectionString,
|
const dataDir = await p.text({
|
||||||
embeddedPostgresDataDir: defaultEmbeddedDir,
|
|
||||||
embeddedPostgresPort: 54329,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const embeddedPostgresDataDir = await p.text({
|
|
||||||
message: "Embedded PostgreSQL data directory",
|
message: "Embedded PostgreSQL data directory",
|
||||||
defaultValue: defaultEmbeddedDir,
|
defaultValue: base.embeddedPostgresDataDir || defaultEmbeddedDir,
|
||||||
placeholder: defaultEmbeddedDir,
|
placeholder: defaultEmbeddedDir,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (p.isCancel(embeddedPostgresDataDir)) {
|
if (p.isCancel(dataDir)) {
|
||||||
p.cancel("Setup cancelled.");
|
p.cancel("Setup cancelled.");
|
||||||
process.exit(0);
|
process.exit(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
const embeddedPostgresPort = await p.text({
|
embeddedPostgresDataDir = dataDir || defaultEmbeddedDir;
|
||||||
|
|
||||||
|
const portValue = await p.text({
|
||||||
message: "Embedded PostgreSQL port",
|
message: "Embedded PostgreSQL port",
|
||||||
defaultValue: "54329",
|
defaultValue: String(base.embeddedPostgresPort || 54329),
|
||||||
placeholder: "54329",
|
placeholder: "54329",
|
||||||
validate: (val) => {
|
validate: (val) => {
|
||||||
const n = Number(val);
|
const n = Number(val);
|
||||||
@@ -62,14 +81,77 @@ export async function promptDatabase(): Promise<DatabaseConfig> {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
if (p.isCancel(embeddedPostgresPort)) {
|
if (p.isCancel(portValue)) {
|
||||||
|
p.cancel("Setup cancelled.");
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
embeddedPostgresPort = Number(portValue || "54329");
|
||||||
|
connectionString = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
const backupEnabled = await p.confirm({
|
||||||
|
message: "Enable automatic database backups?",
|
||||||
|
initialValue: base.backup.enabled,
|
||||||
|
});
|
||||||
|
if (p.isCancel(backupEnabled)) {
|
||||||
|
p.cancel("Setup cancelled.");
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
const backupDirInput = await p.text({
|
||||||
|
message: "Backup directory",
|
||||||
|
defaultValue: base.backup.dir || defaultBackupDir,
|
||||||
|
placeholder: defaultBackupDir,
|
||||||
|
validate: (val) => (!val || val.trim().length === 0 ? "Backup directory is required" : undefined),
|
||||||
|
});
|
||||||
|
if (p.isCancel(backupDirInput)) {
|
||||||
|
p.cancel("Setup cancelled.");
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
const backupIntervalInput = await p.text({
|
||||||
|
message: "Backup interval (minutes)",
|
||||||
|
defaultValue: String(base.backup.intervalMinutes || 60),
|
||||||
|
placeholder: "60",
|
||||||
|
validate: (val) => {
|
||||||
|
const n = Number(val);
|
||||||
|
if (!Number.isInteger(n) || n < 1) return "Interval must be a positive integer";
|
||||||
|
if (n > 10080) return "Interval must be 10080 minutes (7 days) or less";
|
||||||
|
return undefined;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (p.isCancel(backupIntervalInput)) {
|
||||||
|
p.cancel("Setup cancelled.");
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
const backupRetentionInput = await p.text({
|
||||||
|
message: "Backup retention (days)",
|
||||||
|
defaultValue: String(base.backup.retentionDays || 30),
|
||||||
|
placeholder: "30",
|
||||||
|
validate: (val) => {
|
||||||
|
const n = Number(val);
|
||||||
|
if (!Number.isInteger(n) || n < 1) return "Retention must be a positive integer";
|
||||||
|
if (n > 3650) return "Retention must be 3650 days or less";
|
||||||
|
return undefined;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (p.isCancel(backupRetentionInput)) {
|
||||||
p.cancel("Setup cancelled.");
|
p.cancel("Setup cancelled.");
|
||||||
process.exit(0);
|
process.exit(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
mode: "embedded-postgres",
|
mode,
|
||||||
embeddedPostgresDataDir: embeddedPostgresDataDir || defaultEmbeddedDir,
|
connectionString,
|
||||||
embeddedPostgresPort: Number(embeddedPostgresPort || "54329"),
|
embeddedPostgresDataDir,
|
||||||
|
embeddedPostgresPort,
|
||||||
|
backup: {
|
||||||
|
enabled: backupEnabled,
|
||||||
|
intervalMinutes: Number(backupIntervalInput || "60"),
|
||||||
|
retentionDays: Number(backupRetentionInput || "30"),
|
||||||
|
dir: backupDirInput || defaultBackupDir,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -141,6 +141,28 @@ pnpm dev
|
|||||||
|
|
||||||
If you set `DATABASE_URL`, the server will use that instead of embedded PostgreSQL.
|
If you set `DATABASE_URL`, the server will use that instead of embedded PostgreSQL.
|
||||||
|
|
||||||
|
## Automatic DB Backups
|
||||||
|
|
||||||
|
Paperclip can run automatic DB backups on a timer. Defaults:
|
||||||
|
|
||||||
|
- enabled
|
||||||
|
- every 60 minutes
|
||||||
|
- retain 30 days
|
||||||
|
- backup dir: `~/.paperclip/instances/default/data/backups`
|
||||||
|
|
||||||
|
Configure these in:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
pnpm paperclipai configure --section database
|
||||||
|
```
|
||||||
|
|
||||||
|
Environment overrides:
|
||||||
|
|
||||||
|
- `PAPERCLIP_DB_BACKUP_ENABLED=true|false`
|
||||||
|
- `PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES=<minutes>`
|
||||||
|
- `PAPERCLIP_DB_BACKUP_RETENTION_DAYS=<days>`
|
||||||
|
- `PAPERCLIP_DB_BACKUP_DIR=/absolute/or/~/path`
|
||||||
|
|
||||||
## Secrets in Dev
|
## Secrets in Dev
|
||||||
|
|
||||||
Agent env vars now support secret references. By default, secret values are stored with local encryption and only secret refs are persisted in agent config.
|
Agent env vars now support secret references. By default, secret values are stored with local encryption and only secret refs are persisted in agent config.
|
||||||
|
|||||||
333
packages/db/src/backup-lib.ts
Normal file
333
packages/db/src/backup-lib.ts
Normal file
@@ -0,0 +1,333 @@
|
|||||||
|
import { existsSync, mkdirSync, readdirSync, statSync, unlinkSync } from "node:fs";
|
||||||
|
import { writeFile } from "node:fs/promises";
|
||||||
|
import { resolve } from "node:path";
|
||||||
|
import postgres from "postgres";
|
||||||
|
|
||||||
|
export type RunDatabaseBackupOptions = {
|
||||||
|
connectionString: string;
|
||||||
|
backupDir: string;
|
||||||
|
retentionDays: number;
|
||||||
|
filenamePrefix?: string;
|
||||||
|
connectTimeoutSeconds?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type RunDatabaseBackupResult = {
|
||||||
|
backupFile: string;
|
||||||
|
sizeBytes: number;
|
||||||
|
prunedCount: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
function timestamp(date: Date = new Date()): string {
|
||||||
|
const pad = (n: number) => String(n).padStart(2, "0");
|
||||||
|
return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}-${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function pruneOldBackups(backupDir: string, retentionDays: number, filenamePrefix: string): number {
|
||||||
|
if (!existsSync(backupDir)) return 0;
|
||||||
|
const safeRetention = Math.max(1, Math.trunc(retentionDays));
|
||||||
|
const cutoff = Date.now() - safeRetention * 24 * 60 * 60 * 1000;
|
||||||
|
let pruned = 0;
|
||||||
|
|
||||||
|
for (const name of readdirSync(backupDir)) {
|
||||||
|
if (!name.startsWith(`${filenamePrefix}-`) || !name.endsWith(".sql")) continue;
|
||||||
|
const fullPath = resolve(backupDir, name);
|
||||||
|
const stat = statSync(fullPath);
|
||||||
|
if (stat.mtimeMs < cutoff) {
|
||||||
|
unlinkSync(fullPath);
|
||||||
|
pruned++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return pruned;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatBackupSize(sizeBytes: number): string {
|
||||||
|
if (sizeBytes < 1024) return `${sizeBytes}B`;
|
||||||
|
if (sizeBytes < 1024 * 1024) return `${(sizeBytes / 1024).toFixed(1)}K`;
|
||||||
|
return `${(sizeBytes / (1024 * 1024)).toFixed(1)}M`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise<RunDatabaseBackupResult> {
|
||||||
|
const filenamePrefix = opts.filenamePrefix ?? "paperclip";
|
||||||
|
const retentionDays = Math.max(1, Math.trunc(opts.retentionDays));
|
||||||
|
const connectTimeout = Math.max(1, Math.trunc(opts.connectTimeoutSeconds ?? 5));
|
||||||
|
const sql = postgres(opts.connectionString, { max: 1, connect_timeout: connectTimeout });
|
||||||
|
|
||||||
|
try {
|
||||||
|
await sql`SELECT 1`;
|
||||||
|
|
||||||
|
const lines: string[] = [];
|
||||||
|
const emit = (line: string) => lines.push(line);
|
||||||
|
|
||||||
|
emit("-- Paperclip database backup");
|
||||||
|
emit(`-- Created: ${new Date().toISOString()}`);
|
||||||
|
emit("");
|
||||||
|
emit("BEGIN;");
|
||||||
|
emit("");
|
||||||
|
|
||||||
|
// Get all enums
|
||||||
|
const enums = await sql<{ typname: string; labels: string[] }[]>`
|
||||||
|
SELECT t.typname, array_agg(e.enumlabel ORDER BY e.enumsortorder) AS labels
|
||||||
|
FROM pg_type t
|
||||||
|
JOIN pg_enum e ON t.oid = e.enumtypid
|
||||||
|
JOIN pg_namespace n ON t.typnamespace = n.oid
|
||||||
|
WHERE n.nspname = 'public'
|
||||||
|
GROUP BY t.typname
|
||||||
|
ORDER BY t.typname
|
||||||
|
`;
|
||||||
|
|
||||||
|
for (const e of enums) {
|
||||||
|
const labels = e.labels.map((l) => `'${l.replace(/'/g, "''")}'`).join(", ");
|
||||||
|
emit(`CREATE TYPE "public"."${e.typname}" AS ENUM (${labels});`);
|
||||||
|
}
|
||||||
|
if (enums.length > 0) emit("");
|
||||||
|
|
||||||
|
// Get tables in dependency order (referenced tables first)
|
||||||
|
const tables = await sql<{ tablename: string }[]>`
|
||||||
|
SELECT c.relname AS tablename
|
||||||
|
FROM pg_class c
|
||||||
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||||
|
WHERE n.nspname = 'public'
|
||||||
|
AND c.relkind = 'r'
|
||||||
|
AND c.relname != '__drizzle_migrations'
|
||||||
|
ORDER BY c.relname
|
||||||
|
`;
|
||||||
|
|
||||||
|
// Get full CREATE TABLE DDL via column info
|
||||||
|
for (const { tablename } of tables) {
|
||||||
|
const columns = await sql<{
|
||||||
|
column_name: string;
|
||||||
|
data_type: string;
|
||||||
|
udt_name: string;
|
||||||
|
is_nullable: string;
|
||||||
|
column_default: string | null;
|
||||||
|
character_maximum_length: number | null;
|
||||||
|
numeric_precision: number | null;
|
||||||
|
numeric_scale: number | null;
|
||||||
|
}[]>`
|
||||||
|
SELECT column_name, data_type, udt_name, is_nullable, column_default,
|
||||||
|
character_maximum_length, numeric_precision, numeric_scale
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_schema = 'public' AND table_name = ${tablename}
|
||||||
|
ORDER BY ordinal_position
|
||||||
|
`;
|
||||||
|
|
||||||
|
emit(`-- Table: ${tablename}`);
|
||||||
|
emit(`DROP TABLE IF EXISTS "${tablename}" CASCADE;`);
|
||||||
|
|
||||||
|
const colDefs: string[] = [];
|
||||||
|
for (const col of columns) {
|
||||||
|
let typeStr: string;
|
||||||
|
if (col.data_type === "USER-DEFINED") {
|
||||||
|
typeStr = `"${col.udt_name}"`;
|
||||||
|
} else if (col.data_type === "ARRAY") {
|
||||||
|
typeStr = `${col.udt_name.replace(/^_/, "")}[]`;
|
||||||
|
} else if (col.data_type === "character varying") {
|
||||||
|
typeStr = col.character_maximum_length
|
||||||
|
? `varchar(${col.character_maximum_length})`
|
||||||
|
: "varchar";
|
||||||
|
} else if (col.data_type === "numeric" && col.numeric_precision != null) {
|
||||||
|
typeStr =
|
||||||
|
col.numeric_scale != null
|
||||||
|
? `numeric(${col.numeric_precision}, ${col.numeric_scale})`
|
||||||
|
: `numeric(${col.numeric_precision})`;
|
||||||
|
} else {
|
||||||
|
typeStr = col.data_type;
|
||||||
|
}
|
||||||
|
|
||||||
|
let def = ` "${col.column_name}" ${typeStr}`;
|
||||||
|
if (col.column_default != null) def += ` DEFAULT ${col.column_default}`;
|
||||||
|
if (col.is_nullable === "NO") def += " NOT NULL";
|
||||||
|
colDefs.push(def);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Primary key
|
||||||
|
const pk = await sql<{ constraint_name: string; column_names: string[] }[]>`
|
||||||
|
SELECT c.conname AS constraint_name,
|
||||||
|
array_agg(a.attname ORDER BY array_position(c.conkey, a.attnum)) AS column_names
|
||||||
|
FROM pg_constraint c
|
||||||
|
JOIN pg_class t ON t.oid = c.conrelid
|
||||||
|
JOIN pg_namespace n ON n.oid = t.relnamespace
|
||||||
|
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey)
|
||||||
|
WHERE n.nspname = 'public' AND t.relname = ${tablename} AND c.contype = 'p'
|
||||||
|
GROUP BY c.conname
|
||||||
|
`;
|
||||||
|
for (const p of pk) {
|
||||||
|
const cols = p.column_names.map((c) => `"${c}"`).join(", ");
|
||||||
|
colDefs.push(` CONSTRAINT "${p.constraint_name}" PRIMARY KEY (${cols})`);
|
||||||
|
}
|
||||||
|
|
||||||
|
emit(`CREATE TABLE "${tablename}" (`);
|
||||||
|
emit(colDefs.join(",\n"));
|
||||||
|
emit(");");
|
||||||
|
emit("");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Foreign keys (after all tables created)
|
||||||
|
const fks = await sql<{
|
||||||
|
constraint_name: string;
|
||||||
|
source_table: string;
|
||||||
|
source_columns: string[];
|
||||||
|
target_table: string;
|
||||||
|
target_columns: string[];
|
||||||
|
update_rule: string;
|
||||||
|
delete_rule: string;
|
||||||
|
}[]>`
|
||||||
|
SELECT
|
||||||
|
c.conname AS constraint_name,
|
||||||
|
src.relname AS source_table,
|
||||||
|
array_agg(sa.attname ORDER BY array_position(c.conkey, sa.attnum)) AS source_columns,
|
||||||
|
tgt.relname AS target_table,
|
||||||
|
array_agg(ta.attname ORDER BY array_position(c.confkey, ta.attnum)) AS target_columns,
|
||||||
|
CASE c.confupdtype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS update_rule,
|
||||||
|
CASE c.confdeltype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS delete_rule
|
||||||
|
FROM pg_constraint c
|
||||||
|
JOIN pg_class src ON src.oid = c.conrelid
|
||||||
|
JOIN pg_class tgt ON tgt.oid = c.confrelid
|
||||||
|
JOIN pg_namespace n ON n.oid = src.relnamespace
|
||||||
|
JOIN pg_attribute sa ON sa.attrelid = src.oid AND sa.attnum = ANY(c.conkey)
|
||||||
|
JOIN pg_attribute ta ON ta.attrelid = tgt.oid AND ta.attnum = ANY(c.confkey)
|
||||||
|
WHERE c.contype = 'f' AND n.nspname = 'public'
|
||||||
|
GROUP BY c.conname, src.relname, tgt.relname, c.confupdtype, c.confdeltype
|
||||||
|
ORDER BY src.relname, c.conname
|
||||||
|
`;
|
||||||
|
|
||||||
|
if (fks.length > 0) {
|
||||||
|
emit("-- Foreign keys");
|
||||||
|
for (const fk of fks) {
|
||||||
|
const srcCols = fk.source_columns.map((c) => `"${c}"`).join(", ");
|
||||||
|
const tgtCols = fk.target_columns.map((c) => `"${c}"`).join(", ");
|
||||||
|
emit(
|
||||||
|
`ALTER TABLE "${fk.source_table}" ADD CONSTRAINT "${fk.constraint_name}" FOREIGN KEY (${srcCols}) REFERENCES "${fk.target_table}" (${tgtCols}) ON UPDATE ${fk.update_rule} ON DELETE ${fk.delete_rule};`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
emit("");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unique constraints
|
||||||
|
const uniques = await sql<{
|
||||||
|
constraint_name: string;
|
||||||
|
tablename: string;
|
||||||
|
column_names: string[];
|
||||||
|
}[]>`
|
||||||
|
SELECT c.conname AS constraint_name,
|
||||||
|
t.relname AS tablename,
|
||||||
|
array_agg(a.attname ORDER BY array_position(c.conkey, a.attnum)) AS column_names
|
||||||
|
FROM pg_constraint c
|
||||||
|
JOIN pg_class t ON t.oid = c.conrelid
|
||||||
|
JOIN pg_namespace n ON n.oid = t.relnamespace
|
||||||
|
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey)
|
||||||
|
WHERE n.nspname = 'public' AND c.contype = 'u'
|
||||||
|
GROUP BY c.conname, t.relname
|
||||||
|
ORDER BY t.relname, c.conname
|
||||||
|
`;
|
||||||
|
|
||||||
|
if (uniques.length > 0) {
|
||||||
|
emit("-- Unique constraints");
|
||||||
|
for (const u of uniques) {
|
||||||
|
const cols = u.column_names.map((c) => `"${c}"`).join(", ");
|
||||||
|
emit(`ALTER TABLE "${u.tablename}" ADD CONSTRAINT "${u.constraint_name}" UNIQUE (${cols});`);
|
||||||
|
}
|
||||||
|
emit("");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Indexes (non-primary, non-unique-constraint)
|
||||||
|
const indexes = await sql<{ indexdef: string }[]>`
|
||||||
|
SELECT indexdef
|
||||||
|
FROM pg_indexes
|
||||||
|
WHERE schemaname = 'public'
|
||||||
|
AND indexname NOT IN (
|
||||||
|
SELECT conname FROM pg_constraint
|
||||||
|
WHERE connamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public')
|
||||||
|
)
|
||||||
|
ORDER BY tablename, indexname
|
||||||
|
`;
|
||||||
|
|
||||||
|
if (indexes.length > 0) {
|
||||||
|
emit("-- Indexes");
|
||||||
|
for (const idx of indexes) {
|
||||||
|
emit(`${idx.indexdef};`);
|
||||||
|
}
|
||||||
|
emit("");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dump data for each table
|
||||||
|
for (const { tablename } of tables) {
|
||||||
|
const count = await sql<{ n: number }[]>`
|
||||||
|
SELECT count(*)::int AS n FROM ${sql(tablename)}
|
||||||
|
`;
|
||||||
|
if ((count[0]?.n ?? 0) === 0) continue;
|
||||||
|
|
||||||
|
// Get column info for this table
|
||||||
|
const cols = await sql<{ column_name: string; data_type: string }[]>`
|
||||||
|
SELECT column_name, data_type
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_schema = 'public' AND table_name = ${tablename}
|
||||||
|
ORDER BY ordinal_position
|
||||||
|
`;
|
||||||
|
const colNames = cols.map((c) => `"${c.column_name}"`).join(", ");
|
||||||
|
|
||||||
|
emit(`-- Data for: ${tablename} (${count[0]!.n} rows)`);
|
||||||
|
|
||||||
|
const rows = await sql`SELECT * FROM ${sql(tablename)}`.values();
|
||||||
|
for (const row of rows) {
|
||||||
|
const values = row.map((val: unknown) => {
|
||||||
|
if (val === null || val === undefined) return "NULL";
|
||||||
|
if (typeof val === "boolean") return val ? "true" : "false";
|
||||||
|
if (typeof val === "number") return String(val);
|
||||||
|
if (val instanceof Date) return `'${val.toISOString()}'`;
|
||||||
|
if (typeof val === "object") return `'${JSON.stringify(val).replace(/'/g, "''")}'`;
|
||||||
|
return `'${String(val).replace(/'/g, "''")}'`;
|
||||||
|
});
|
||||||
|
emit(`INSERT INTO "${tablename}" (${colNames}) VALUES (${values.join(", ")});`);
|
||||||
|
}
|
||||||
|
emit("");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sequence values
|
||||||
|
const sequences = await sql<{ sequence_name: string }[]>`
|
||||||
|
SELECT sequence_name
|
||||||
|
FROM information_schema.sequences
|
||||||
|
WHERE sequence_schema = 'public'
|
||||||
|
ORDER BY sequence_name
|
||||||
|
`;
|
||||||
|
|
||||||
|
if (sequences.length > 0) {
|
||||||
|
emit("-- Sequence values");
|
||||||
|
for (const seq of sequences) {
|
||||||
|
const val = await sql<{ last_value: string }[]>`
|
||||||
|
SELECT last_value::text FROM ${sql(seq.sequence_name)}
|
||||||
|
`;
|
||||||
|
if (val[0]) {
|
||||||
|
emit(`SELECT setval('"${seq.sequence_name}"', ${val[0].last_value});`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
emit("");
|
||||||
|
}
|
||||||
|
|
||||||
|
emit("COMMIT;");
|
||||||
|
emit("");
|
||||||
|
|
||||||
|
// Write the backup file
|
||||||
|
mkdirSync(opts.backupDir, { recursive: true });
|
||||||
|
const backupFile = resolve(opts.backupDir, `${filenamePrefix}-${timestamp()}.sql`);
|
||||||
|
await writeFile(backupFile, lines.join("\n"), "utf8");
|
||||||
|
|
||||||
|
const sizeBytes = statSync(backupFile).size;
|
||||||
|
const prunedCount = pruneOldBackups(opts.backupDir, retentionDays, filenamePrefix);
|
||||||
|
|
||||||
|
return {
|
||||||
|
backupFile,
|
||||||
|
sizeBytes,
|
||||||
|
prunedCount,
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
await sql.end();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatDatabaseBackupResult(result: RunDatabaseBackupResult): string {
|
||||||
|
const size = formatBackupSize(result.sizeBytes);
|
||||||
|
const pruned = result.prunedCount > 0 ? `; pruned ${result.prunedCount} old backup(s)` : "";
|
||||||
|
return `${result.backupFile} (${size}${pruned})`;
|
||||||
|
}
|
||||||
@@ -1,338 +1,122 @@
|
|||||||
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, unlinkSync } from "node:fs";
|
import { existsSync, readFileSync } from "node:fs";
|
||||||
import { writeFile } from "node:fs/promises";
|
import os from "node:os";
|
||||||
import { resolve } from "node:path";
|
import path from "node:path";
|
||||||
import postgres from "postgres";
|
import { formatDatabaseBackupResult, runDatabaseBackup } from "./backup-lib.js";
|
||||||
|
|
||||||
const PROJECT_ROOT = resolve(import.meta.dirname, "../../..");
|
type PartialConfig = {
|
||||||
const BACKUP_DIR = resolve(PROJECT_ROOT, "data/backups");
|
database?: {
|
||||||
const CONFIG_FILE = resolve(PROJECT_ROOT, ".paperclip/config.json");
|
mode?: "embedded-postgres" | "postgres";
|
||||||
const MAX_AGE_DAYS = 30;
|
connectionString?: string;
|
||||||
|
embeddedPostgresPort?: number;
|
||||||
|
backup?: {
|
||||||
|
dir?: string;
|
||||||
|
retentionDays?: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
function loadPort(): number {
|
function expandHomePrefix(value: string): string {
|
||||||
|
if (value === "~") return os.homedir();
|
||||||
|
if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2));
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolvePaperclipHomeDir(): string {
|
||||||
|
const envHome = process.env.PAPERCLIP_HOME?.trim();
|
||||||
|
if (envHome) return path.resolve(expandHomePrefix(envHome));
|
||||||
|
return path.resolve(os.homedir(), ".paperclip");
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolvePaperclipInstanceId(): string {
|
||||||
|
const raw = process.env.PAPERCLIP_INSTANCE_ID?.trim() || "default";
|
||||||
|
if (!/^[a-zA-Z0-9_-]+$/.test(raw)) {
|
||||||
|
throw new Error(`Invalid PAPERCLIP_INSTANCE_ID '${raw}'.`);
|
||||||
|
}
|
||||||
|
return raw;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveDefaultConfigPath(): string {
|
||||||
|
return path.resolve(resolvePaperclipHomeDir(), "instances", resolvePaperclipInstanceId(), "config.json");
|
||||||
|
}
|
||||||
|
|
||||||
|
function readConfig(configPath: string): PartialConfig | null {
|
||||||
|
if (!existsSync(configPath)) return null;
|
||||||
try {
|
try {
|
||||||
const raw = readFileSync(CONFIG_FILE, "utf8");
|
const parsed = JSON.parse(readFileSync(configPath, "utf8"));
|
||||||
const config = JSON.parse(raw);
|
return typeof parsed === "object" && parsed ? (parsed as PartialConfig) : null;
|
||||||
const port = config?.database?.embeddedPostgresPort;
|
} catch {
|
||||||
if (typeof port === "number" && Number.isFinite(port)) return port;
|
return null;
|
||||||
} catch {}
|
}
|
||||||
return 54329;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function timestamp(): string {
|
function asPositiveInt(value: unknown): number | null {
|
||||||
const d = new Date();
|
if (typeof value !== "number" || !Number.isFinite(value)) return null;
|
||||||
const pad = (n: number) => String(n).padStart(2, "0");
|
const rounded = Math.trunc(value);
|
||||||
return `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}-${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}`;
|
return rounded > 0 ? rounded : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function pruneOldBackups() {
|
function resolveEmbeddedPort(config: PartialConfig | null): number {
|
||||||
if (!existsSync(BACKUP_DIR)) return;
|
return asPositiveInt(config?.database?.embeddedPostgresPort) ?? 54329;
|
||||||
const cutoff = Date.now() - MAX_AGE_DAYS * 24 * 60 * 60 * 1000;
|
|
||||||
let pruned = 0;
|
|
||||||
for (const name of readdirSync(BACKUP_DIR)) {
|
|
||||||
if (!name.startsWith("paperclip-") || !name.endsWith(".sql")) continue;
|
|
||||||
const fullPath = resolve(BACKUP_DIR, name);
|
|
||||||
const stat = statSync(fullPath);
|
|
||||||
if (stat.mtimeMs < cutoff) {
|
|
||||||
unlinkSync(fullPath);
|
|
||||||
pruned++;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function resolveConnectionString(config: PartialConfig | null): string {
|
||||||
|
const envUrl = process.env.DATABASE_URL?.trim();
|
||||||
|
if (envUrl) return envUrl;
|
||||||
|
|
||||||
|
if (config?.database?.mode === "postgres" && typeof config.database.connectionString === "string") {
|
||||||
|
const trimmed = config.database.connectionString.trim();
|
||||||
|
if (trimmed) return trimmed;
|
||||||
}
|
}
|
||||||
if (pruned > 0) console.log(`Pruned ${pruned} backup(s) older than ${MAX_AGE_DAYS} days.`);
|
|
||||||
|
const port = resolveEmbeddedPort(config);
|
||||||
|
return `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveDefaultBackupDir(): string {
|
||||||
|
return path.resolve(resolvePaperclipHomeDir(), "instances", resolvePaperclipInstanceId(), "data", "backups");
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveBackupDir(config: PartialConfig | null): string {
|
||||||
|
const raw = config?.database?.backup?.dir;
|
||||||
|
if (typeof raw === "string" && raw.trim().length > 0) {
|
||||||
|
return path.resolve(expandHomePrefix(raw.trim()));
|
||||||
|
}
|
||||||
|
return resolveDefaultBackupDir();
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveRetentionDays(config: PartialConfig | null): number {
|
||||||
|
return asPositiveInt(config?.database?.backup?.retentionDays) ?? 30;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function main() {
|
async function main() {
|
||||||
const port = loadPort();
|
const configPath = resolveDefaultConfigPath();
|
||||||
const connString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`;
|
const config = readConfig(configPath);
|
||||||
|
const connectionString = resolveConnectionString(config);
|
||||||
|
const backupDir = resolveBackupDir(config);
|
||||||
|
const retentionDays = resolveRetentionDays(config);
|
||||||
|
|
||||||
console.log(`Connecting to embedded PostgreSQL on port ${port}...`);
|
console.log(`Config path: ${configPath}`);
|
||||||
|
console.log(`Backing up database to: ${backupDir}`);
|
||||||
const sql = postgres(connString, { max: 1, connect_timeout: 5 });
|
console.log(`Retention window: ${retentionDays} day(s)`);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Verify connection
|
const result = await runDatabaseBackup({
|
||||||
await sql`SELECT 1`;
|
connectionString,
|
||||||
} catch (err: any) {
|
backupDir,
|
||||||
console.error(`Error: Cannot connect to embedded PostgreSQL on port ${port}.`);
|
retentionDays,
|
||||||
console.error(" Make sure the server is running (pnpm dev).");
|
filenamePrefix: "paperclip",
|
||||||
process.exit(1);
|
});
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
console.log(`Backup saved: ${formatDatabaseBackupResult(result)}`);
|
||||||
const lines: string[] = [];
|
} catch (err) {
|
||||||
const emit = (line: string) => lines.push(line);
|
console.error("Backup failed.");
|
||||||
|
if (err instanceof Error) {
|
||||||
emit("-- Paperclip database backup");
|
console.error(err.message);
|
||||||
emit(`-- Created: ${new Date().toISOString()}`);
|
|
||||||
emit(`-- Server port: ${port}`);
|
|
||||||
emit("");
|
|
||||||
emit("BEGIN;");
|
|
||||||
emit("");
|
|
||||||
|
|
||||||
// Get all enums
|
|
||||||
const enums = await sql<{ typname: string; labels: string[] }[]>`
|
|
||||||
SELECT t.typname, array_agg(e.enumlabel ORDER BY e.enumsortorder) AS labels
|
|
||||||
FROM pg_type t
|
|
||||||
JOIN pg_enum e ON t.oid = e.enumtypid
|
|
||||||
JOIN pg_namespace n ON t.typnamespace = n.oid
|
|
||||||
WHERE n.nspname = 'public'
|
|
||||||
GROUP BY t.typname
|
|
||||||
ORDER BY t.typname
|
|
||||||
`;
|
|
||||||
|
|
||||||
for (const e of enums) {
|
|
||||||
const labels = e.labels.map((l) => `'${l.replace(/'/g, "''")}'`).join(", ");
|
|
||||||
emit(`CREATE TYPE "public"."${e.typname}" AS ENUM (${labels});`);
|
|
||||||
}
|
|
||||||
if (enums.length > 0) emit("");
|
|
||||||
|
|
||||||
// Get tables in dependency order (referenced tables first)
|
|
||||||
const tables = await sql<{ tablename: string }[]>`
|
|
||||||
SELECT c.relname AS tablename
|
|
||||||
FROM pg_class c
|
|
||||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
||||||
WHERE n.nspname = 'public'
|
|
||||||
AND c.relkind = 'r'
|
|
||||||
AND c.relname != '__drizzle_migrations'
|
|
||||||
ORDER BY c.relname
|
|
||||||
`;
|
|
||||||
|
|
||||||
// Get full CREATE TABLE DDL via column info
|
|
||||||
for (const { tablename } of tables) {
|
|
||||||
const columns = await sql<{
|
|
||||||
column_name: string;
|
|
||||||
data_type: string;
|
|
||||||
udt_name: string;
|
|
||||||
is_nullable: string;
|
|
||||||
column_default: string | null;
|
|
||||||
character_maximum_length: number | null;
|
|
||||||
numeric_precision: number | null;
|
|
||||||
numeric_scale: number | null;
|
|
||||||
}[]>`
|
|
||||||
SELECT column_name, data_type, udt_name, is_nullable, column_default,
|
|
||||||
character_maximum_length, numeric_precision, numeric_scale
|
|
||||||
FROM information_schema.columns
|
|
||||||
WHERE table_schema = 'public' AND table_name = ${tablename}
|
|
||||||
ORDER BY ordinal_position
|
|
||||||
`;
|
|
||||||
|
|
||||||
emit(`-- Table: ${tablename}`);
|
|
||||||
emit(`DROP TABLE IF EXISTS "${tablename}" CASCADE;`);
|
|
||||||
|
|
||||||
const colDefs: string[] = [];
|
|
||||||
for (const col of columns) {
|
|
||||||
let typeStr: string;
|
|
||||||
if (col.data_type === "USER-DEFINED") {
|
|
||||||
typeStr = `"${col.udt_name}"`;
|
|
||||||
} else if (col.data_type === "ARRAY") {
|
|
||||||
typeStr = `${col.udt_name.replace(/^_/, "")}[]`;
|
|
||||||
} else if (col.data_type === "character varying") {
|
|
||||||
typeStr = col.character_maximum_length
|
|
||||||
? `varchar(${col.character_maximum_length})`
|
|
||||||
: "varchar";
|
|
||||||
} else if (col.data_type === "numeric" && col.numeric_precision != null) {
|
|
||||||
typeStr =
|
|
||||||
col.numeric_scale != null
|
|
||||||
? `numeric(${col.numeric_precision}, ${col.numeric_scale})`
|
|
||||||
: `numeric(${col.numeric_precision})`;
|
|
||||||
} else {
|
} else {
|
||||||
typeStr = col.data_type;
|
console.error(String(err));
|
||||||
}
|
}
|
||||||
|
|
||||||
let def = ` "${col.column_name}" ${typeStr}`;
|
|
||||||
if (col.column_default != null) def += ` DEFAULT ${col.column_default}`;
|
|
||||||
if (col.is_nullable === "NO") def += " NOT NULL";
|
|
||||||
colDefs.push(def);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Primary key
|
|
||||||
const pk = await sql<{ constraint_name: string; column_names: string[] }[]>`
|
|
||||||
SELECT c.conname AS constraint_name,
|
|
||||||
array_agg(a.attname ORDER BY array_position(c.conkey, a.attnum)) AS column_names
|
|
||||||
FROM pg_constraint c
|
|
||||||
JOIN pg_class t ON t.oid = c.conrelid
|
|
||||||
JOIN pg_namespace n ON n.oid = t.relnamespace
|
|
||||||
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey)
|
|
||||||
WHERE n.nspname = 'public' AND t.relname = ${tablename} AND c.contype = 'p'
|
|
||||||
GROUP BY c.conname
|
|
||||||
`;
|
|
||||||
for (const p of pk) {
|
|
||||||
const cols = p.column_names.map((c) => `"${c}"`).join(", ");
|
|
||||||
colDefs.push(` CONSTRAINT "${p.constraint_name}" PRIMARY KEY (${cols})`);
|
|
||||||
}
|
|
||||||
|
|
||||||
emit(`CREATE TABLE "${tablename}" (`);
|
|
||||||
emit(colDefs.join(",\n"));
|
|
||||||
emit(");");
|
|
||||||
emit("");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Foreign keys (after all tables created)
|
|
||||||
const fks = await sql<{
|
|
||||||
constraint_name: string;
|
|
||||||
source_table: string;
|
|
||||||
source_columns: string[];
|
|
||||||
target_table: string;
|
|
||||||
target_columns: string[];
|
|
||||||
update_rule: string;
|
|
||||||
delete_rule: string;
|
|
||||||
}[]>`
|
|
||||||
SELECT
|
|
||||||
c.conname AS constraint_name,
|
|
||||||
src.relname AS source_table,
|
|
||||||
array_agg(sa.attname ORDER BY array_position(c.conkey, sa.attnum)) AS source_columns,
|
|
||||||
tgt.relname AS target_table,
|
|
||||||
array_agg(ta.attname ORDER BY array_position(c.confkey, ta.attnum)) AS target_columns,
|
|
||||||
CASE c.confupdtype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS update_rule,
|
|
||||||
CASE c.confdeltype WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END AS delete_rule
|
|
||||||
FROM pg_constraint c
|
|
||||||
JOIN pg_class src ON src.oid = c.conrelid
|
|
||||||
JOIN pg_class tgt ON tgt.oid = c.confrelid
|
|
||||||
JOIN pg_namespace n ON n.oid = src.relnamespace
|
|
||||||
JOIN pg_attribute sa ON sa.attrelid = src.oid AND sa.attnum = ANY(c.conkey)
|
|
||||||
JOIN pg_attribute ta ON ta.attrelid = tgt.oid AND ta.attnum = ANY(c.confkey)
|
|
||||||
WHERE c.contype = 'f' AND n.nspname = 'public'
|
|
||||||
GROUP BY c.conname, src.relname, tgt.relname, c.confupdtype, c.confdeltype
|
|
||||||
ORDER BY src.relname, c.conname
|
|
||||||
`;
|
|
||||||
|
|
||||||
if (fks.length > 0) {
|
|
||||||
emit("-- Foreign keys");
|
|
||||||
for (const fk of fks) {
|
|
||||||
const srcCols = fk.source_columns.map((c) => `"${c}"`).join(", ");
|
|
||||||
const tgtCols = fk.target_columns.map((c) => `"${c}"`).join(", ");
|
|
||||||
emit(
|
|
||||||
`ALTER TABLE "${fk.source_table}" ADD CONSTRAINT "${fk.constraint_name}" FOREIGN KEY (${srcCols}) REFERENCES "${fk.target_table}" (${tgtCols}) ON UPDATE ${fk.update_rule} ON DELETE ${fk.delete_rule};`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
emit("");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Unique constraints
|
|
||||||
const uniques = await sql<{
|
|
||||||
constraint_name: string;
|
|
||||||
tablename: string;
|
|
||||||
column_names: string[];
|
|
||||||
}[]>`
|
|
||||||
SELECT c.conname AS constraint_name,
|
|
||||||
t.relname AS tablename,
|
|
||||||
array_agg(a.attname ORDER BY array_position(c.conkey, a.attnum)) AS column_names
|
|
||||||
FROM pg_constraint c
|
|
||||||
JOIN pg_class t ON t.oid = c.conrelid
|
|
||||||
JOIN pg_namespace n ON n.oid = t.relnamespace
|
|
||||||
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey)
|
|
||||||
WHERE n.nspname = 'public' AND c.contype = 'u'
|
|
||||||
GROUP BY c.conname, t.relname
|
|
||||||
ORDER BY t.relname, c.conname
|
|
||||||
`;
|
|
||||||
|
|
||||||
if (uniques.length > 0) {
|
|
||||||
emit("-- Unique constraints");
|
|
||||||
for (const u of uniques) {
|
|
||||||
const cols = u.column_names.map((c) => `"${c}"`).join(", ");
|
|
||||||
emit(`ALTER TABLE "${u.tablename}" ADD CONSTRAINT "${u.constraint_name}" UNIQUE (${cols});`);
|
|
||||||
}
|
|
||||||
emit("");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Indexes (non-primary, non-unique-constraint)
|
|
||||||
const indexes = await sql<{ indexdef: string }[]>`
|
|
||||||
SELECT indexdef
|
|
||||||
FROM pg_indexes
|
|
||||||
WHERE schemaname = 'public'
|
|
||||||
AND indexname NOT IN (
|
|
||||||
SELECT conname FROM pg_constraint
|
|
||||||
WHERE connamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public')
|
|
||||||
)
|
|
||||||
ORDER BY tablename, indexname
|
|
||||||
`;
|
|
||||||
|
|
||||||
if (indexes.length > 0) {
|
|
||||||
emit("-- Indexes");
|
|
||||||
for (const idx of indexes) {
|
|
||||||
emit(`${idx.indexdef};`);
|
|
||||||
}
|
|
||||||
emit("");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dump data for each table
|
|
||||||
for (const { tablename } of tables) {
|
|
||||||
const count = await sql<{ n: number }[]>`
|
|
||||||
SELECT count(*)::int AS n FROM ${sql(tablename)}
|
|
||||||
`;
|
|
||||||
if ((count[0]?.n ?? 0) === 0) continue;
|
|
||||||
|
|
||||||
// Get column info for this table
|
|
||||||
const cols = await sql<{ column_name: string; data_type: string }[]>`
|
|
||||||
SELECT column_name, data_type
|
|
||||||
FROM information_schema.columns
|
|
||||||
WHERE table_schema = 'public' AND table_name = ${tablename}
|
|
||||||
ORDER BY ordinal_position
|
|
||||||
`;
|
|
||||||
const colNames = cols.map((c) => `"${c.column_name}"`).join(", ");
|
|
||||||
|
|
||||||
emit(`-- Data for: ${tablename} (${count[0]!.n} rows)`);
|
|
||||||
|
|
||||||
const rows = await sql`SELECT * FROM ${sql(tablename)}`.values();
|
|
||||||
for (const row of rows) {
|
|
||||||
const values = row.map((val: any) => {
|
|
||||||
if (val === null || val === undefined) return "NULL";
|
|
||||||
if (typeof val === "boolean") return val ? "true" : "false";
|
|
||||||
if (typeof val === "number") return String(val);
|
|
||||||
if (val instanceof Date) return `'${val.toISOString()}'`;
|
|
||||||
if (typeof val === "object") return `'${JSON.stringify(val).replace(/'/g, "''")}'`;
|
|
||||||
return `'${String(val).replace(/'/g, "''")}'`;
|
|
||||||
});
|
|
||||||
emit(`INSERT INTO "${tablename}" (${colNames}) VALUES (${values.join(", ")});`);
|
|
||||||
}
|
|
||||||
emit("");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sequence values
|
|
||||||
const sequences = await sql<{ sequence_name: string }[]>`
|
|
||||||
SELECT sequence_name
|
|
||||||
FROM information_schema.sequences
|
|
||||||
WHERE sequence_schema = 'public'
|
|
||||||
ORDER BY sequence_name
|
|
||||||
`;
|
|
||||||
|
|
||||||
if (sequences.length > 0) {
|
|
||||||
emit("-- Sequence values");
|
|
||||||
for (const seq of sequences) {
|
|
||||||
const val = await sql<{ last_value: string }[]>`
|
|
||||||
SELECT last_value::text FROM ${sql(seq.sequence_name)}
|
|
||||||
`;
|
|
||||||
if (val[0]) {
|
|
||||||
emit(`SELECT setval('"${seq.sequence_name}"', ${val[0].last_value});`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
emit("");
|
|
||||||
}
|
|
||||||
|
|
||||||
emit("COMMIT;");
|
|
||||||
emit("");
|
|
||||||
|
|
||||||
// Write the backup file
|
|
||||||
mkdirSync(BACKUP_DIR, { recursive: true });
|
|
||||||
const backupFile = resolve(BACKUP_DIR, `paperclip-${timestamp()}.sql`);
|
|
||||||
await writeFile(backupFile, lines.join("\n"), "utf8");
|
|
||||||
|
|
||||||
const sizeBytes = statSync(backupFile).size;
|
|
||||||
const sizeStr =
|
|
||||||
sizeBytes < 1024
|
|
||||||
? `${sizeBytes}B`
|
|
||||||
: sizeBytes < 1024 * 1024
|
|
||||||
? `${(sizeBytes / 1024).toFixed(1)}K`
|
|
||||||
: `${(sizeBytes / (1024 * 1024)).toFixed(1)}M`;
|
|
||||||
|
|
||||||
console.log(`Backup saved: ${backupFile} (${sizeStr})`);
|
|
||||||
|
|
||||||
pruneOldBackups();
|
|
||||||
} finally {
|
|
||||||
await sql.end();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
main().catch((err) => {
|
|
||||||
console.error(err);
|
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
});
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await main();
|
||||||
|
|||||||
@@ -10,4 +10,10 @@ export {
|
|||||||
type MigrationBootstrapResult,
|
type MigrationBootstrapResult,
|
||||||
type Db,
|
type Db,
|
||||||
} from "./client.js";
|
} from "./client.js";
|
||||||
|
export {
|
||||||
|
runDatabaseBackup,
|
||||||
|
formatDatabaseBackupResult,
|
||||||
|
type RunDatabaseBackupOptions,
|
||||||
|
type RunDatabaseBackupResult,
|
||||||
|
} from "./backup-lib.js";
|
||||||
export * from "./schema/index.js";
|
export * from "./schema/index.js";
|
||||||
|
|||||||
@@ -18,11 +18,24 @@ export const llmConfigSchema = z.object({
|
|||||||
apiKey: z.string().optional(),
|
apiKey: z.string().optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
export const databaseBackupConfigSchema = z.object({
|
||||||
|
enabled: z.boolean().default(true),
|
||||||
|
intervalMinutes: z.number().int().min(1).max(7 * 24 * 60).default(60),
|
||||||
|
retentionDays: z.number().int().min(1).max(3650).default(30),
|
||||||
|
dir: z.string().default("~/.paperclip/instances/default/data/backups"),
|
||||||
|
});
|
||||||
|
|
||||||
export const databaseConfigSchema = z.object({
|
export const databaseConfigSchema = z.object({
|
||||||
mode: z.enum(["embedded-postgres", "postgres"]).default("embedded-postgres"),
|
mode: z.enum(["embedded-postgres", "postgres"]).default("embedded-postgres"),
|
||||||
connectionString: z.string().optional(),
|
connectionString: z.string().optional(),
|
||||||
embeddedPostgresDataDir: z.string().default("~/.paperclip/instances/default/db"),
|
embeddedPostgresDataDir: z.string().default("~/.paperclip/instances/default/db"),
|
||||||
embeddedPostgresPort: z.number().int().min(1).max(65535).default(54329),
|
embeddedPostgresPort: z.number().int().min(1).max(65535).default(54329),
|
||||||
|
backup: databaseBackupConfigSchema.default({
|
||||||
|
enabled: true,
|
||||||
|
intervalMinutes: 60,
|
||||||
|
retentionDays: 30,
|
||||||
|
dir: "~/.paperclip/instances/default/data/backups",
|
||||||
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
export const loggingConfigSchema = z.object({
|
export const loggingConfigSchema = z.object({
|
||||||
@@ -160,3 +173,4 @@ export type SecretsConfig = z.infer<typeof secretsConfigSchema>;
|
|||||||
export type SecretsLocalEncryptedConfig = z.infer<typeof secretsLocalEncryptedConfigSchema>;
|
export type SecretsLocalEncryptedConfig = z.infer<typeof secretsLocalEncryptedConfigSchema>;
|
||||||
export type AuthConfig = z.infer<typeof authConfigSchema>;
|
export type AuthConfig = z.infer<typeof authConfigSchema>;
|
||||||
export type ConfigMeta = z.infer<typeof configMetaSchema>;
|
export type ConfigMeta = z.infer<typeof configMetaSchema>;
|
||||||
|
export type DatabaseBackupConfig = z.infer<typeof databaseBackupConfigSchema>;
|
||||||
|
|||||||
@@ -243,6 +243,7 @@ export {
|
|||||||
paperclipConfigSchema,
|
paperclipConfigSchema,
|
||||||
configMetaSchema,
|
configMetaSchema,
|
||||||
llmConfigSchema,
|
llmConfigSchema,
|
||||||
|
databaseBackupConfigSchema,
|
||||||
databaseConfigSchema,
|
databaseConfigSchema,
|
||||||
loggingConfigSchema,
|
loggingConfigSchema,
|
||||||
serverConfigSchema,
|
serverConfigSchema,
|
||||||
@@ -254,6 +255,7 @@ export {
|
|||||||
secretsLocalEncryptedConfigSchema,
|
secretsLocalEncryptedConfigSchema,
|
||||||
type PaperclipConfig,
|
type PaperclipConfig,
|
||||||
type LlmConfig,
|
type LlmConfig,
|
||||||
|
type DatabaseBackupConfig,
|
||||||
type DatabaseConfig,
|
type DatabaseConfig,
|
||||||
type LoggingConfig,
|
type LoggingConfig,
|
||||||
type ServerConfig,
|
type ServerConfig,
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
# Backup the embedded PostgreSQL database to data/backups/
|
# Backup the configured Paperclip database to the configured backup directory
|
||||||
|
# (default: ~/.paperclip/instances/<instance-id>/data/backups)
|
||||||
#
|
#
|
||||||
# Usage:
|
# Usage:
|
||||||
# ./scripts/backup-db.sh
|
# ./scripts/backup-db.sh
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import {
|
|||||||
type StorageProvider,
|
type StorageProvider,
|
||||||
} from "@paperclipai/shared";
|
} from "@paperclipai/shared";
|
||||||
import {
|
import {
|
||||||
|
resolveDefaultBackupDir,
|
||||||
resolveDefaultEmbeddedPostgresDir,
|
resolveDefaultEmbeddedPostgresDir,
|
||||||
resolveDefaultSecretsKeyFilePath,
|
resolveDefaultSecretsKeyFilePath,
|
||||||
resolveDefaultStorageDir,
|
resolveDefaultStorageDir,
|
||||||
@@ -40,6 +41,10 @@ export interface Config {
|
|||||||
databaseUrl: string | undefined;
|
databaseUrl: string | undefined;
|
||||||
embeddedPostgresDataDir: string;
|
embeddedPostgresDataDir: string;
|
||||||
embeddedPostgresPort: number;
|
embeddedPostgresPort: number;
|
||||||
|
databaseBackupEnabled: boolean;
|
||||||
|
databaseBackupIntervalMinutes: number;
|
||||||
|
databaseBackupRetentionDays: number;
|
||||||
|
databaseBackupDir: string;
|
||||||
serveUi: boolean;
|
serveUi: boolean;
|
||||||
uiDevMiddleware: boolean;
|
uiDevMiddleware: boolean;
|
||||||
secretsProvider: SecretProvider;
|
secretsProvider: SecretProvider;
|
||||||
@@ -66,6 +71,7 @@ export function loadConfig(): Config {
|
|||||||
fileDatabaseMode === "postgres"
|
fileDatabaseMode === "postgres"
|
||||||
? fileConfig?.database.connectionString
|
? fileConfig?.database.connectionString
|
||||||
: undefined;
|
: undefined;
|
||||||
|
const fileDatabaseBackup = fileConfig?.database.backup;
|
||||||
const fileSecrets = fileConfig?.secrets;
|
const fileSecrets = fileConfig?.secrets;
|
||||||
const fileStorage = fileConfig?.storage;
|
const fileStorage = fileConfig?.storage;
|
||||||
const strictModeFromEnv = process.env.PAPERCLIP_SECRETS_STRICT_MODE;
|
const strictModeFromEnv = process.env.PAPERCLIP_SECRETS_STRICT_MODE;
|
||||||
@@ -148,6 +154,27 @@ export function loadConfig(): Config {
|
|||||||
companyDeletionEnvRaw !== undefined
|
companyDeletionEnvRaw !== undefined
|
||||||
? companyDeletionEnvRaw === "true"
|
? companyDeletionEnvRaw === "true"
|
||||||
: deploymentMode === "local_trusted";
|
: deploymentMode === "local_trusted";
|
||||||
|
const databaseBackupEnabled =
|
||||||
|
process.env.PAPERCLIP_DB_BACKUP_ENABLED !== undefined
|
||||||
|
? process.env.PAPERCLIP_DB_BACKUP_ENABLED === "true"
|
||||||
|
: (fileDatabaseBackup?.enabled ?? true);
|
||||||
|
const databaseBackupIntervalMinutes = Math.max(
|
||||||
|
1,
|
||||||
|
Number(process.env.PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES) ||
|
||||||
|
fileDatabaseBackup?.intervalMinutes ||
|
||||||
|
60,
|
||||||
|
);
|
||||||
|
const databaseBackupRetentionDays = Math.max(
|
||||||
|
1,
|
||||||
|
Number(process.env.PAPERCLIP_DB_BACKUP_RETENTION_DAYS) ||
|
||||||
|
fileDatabaseBackup?.retentionDays ||
|
||||||
|
30,
|
||||||
|
);
|
||||||
|
const databaseBackupDir = resolveHomeAwarePath(
|
||||||
|
process.env.PAPERCLIP_DB_BACKUP_DIR ??
|
||||||
|
fileDatabaseBackup?.dir ??
|
||||||
|
resolveDefaultBackupDir(),
|
||||||
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
deploymentMode,
|
deploymentMode,
|
||||||
@@ -163,6 +190,10 @@ export function loadConfig(): Config {
|
|||||||
fileConfig?.database.embeddedPostgresDataDir ?? resolveDefaultEmbeddedPostgresDir(),
|
fileConfig?.database.embeddedPostgresDataDir ?? resolveDefaultEmbeddedPostgresDir(),
|
||||||
),
|
),
|
||||||
embeddedPostgresPort: fileConfig?.database.embeddedPostgresPort ?? 54329,
|
embeddedPostgresPort: fileConfig?.database.embeddedPostgresPort ?? 54329,
|
||||||
|
databaseBackupEnabled,
|
||||||
|
databaseBackupIntervalMinutes,
|
||||||
|
databaseBackupRetentionDays,
|
||||||
|
databaseBackupDir,
|
||||||
serveUi:
|
serveUi:
|
||||||
process.env.SERVE_UI !== undefined
|
process.env.SERVE_UI !== undefined
|
||||||
? process.env.SERVE_UI === "true"
|
? process.env.SERVE_UI === "true"
|
||||||
|
|||||||
@@ -49,6 +49,10 @@ export function resolveDefaultStorageDir(): string {
|
|||||||
return path.resolve(resolvePaperclipInstanceRoot(), "data", "storage");
|
return path.resolve(resolvePaperclipInstanceRoot(), "data", "storage");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function resolveDefaultBackupDir(): string {
|
||||||
|
return path.resolve(resolvePaperclipInstanceRoot(), "data", "backups");
|
||||||
|
}
|
||||||
|
|
||||||
export function resolveDefaultAgentWorkspaceDir(agentId: string): string {
|
export function resolveDefaultAgentWorkspaceDir(agentId: string): string {
|
||||||
const trimmed = agentId.trim();
|
const trimmed = agentId.trim();
|
||||||
if (!PATH_SEGMENT_RE.test(trimmed)) {
|
if (!PATH_SEGMENT_RE.test(trimmed)) {
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ import {
|
|||||||
inspectMigrations,
|
inspectMigrations,
|
||||||
applyPendingMigrations,
|
applyPendingMigrations,
|
||||||
reconcilePendingMigrationHistory,
|
reconcilePendingMigrationHistory,
|
||||||
|
formatDatabaseBackupResult,
|
||||||
|
runDatabaseBackup,
|
||||||
authUsers,
|
authUsers,
|
||||||
companies,
|
companies,
|
||||||
companyMemberships,
|
companyMemberships,
|
||||||
@@ -220,6 +222,7 @@ let db;
|
|||||||
let embeddedPostgres: EmbeddedPostgresInstance | null = null;
|
let embeddedPostgres: EmbeddedPostgresInstance | null = null;
|
||||||
let embeddedPostgresStartedByThisProcess = false;
|
let embeddedPostgresStartedByThisProcess = false;
|
||||||
let migrationSummary: MigrationSummary = "skipped";
|
let migrationSummary: MigrationSummary = "skipped";
|
||||||
|
let activeDatabaseConnectionString: string;
|
||||||
let startupDbInfo:
|
let startupDbInfo:
|
||||||
| { mode: "external-postgres"; connectionString: string }
|
| { mode: "external-postgres"; connectionString: string }
|
||||||
| { mode: "embedded-postgres"; dataDir: string; port: number };
|
| { mode: "embedded-postgres"; dataDir: string; port: number };
|
||||||
@@ -228,6 +231,7 @@ if (config.databaseUrl) {
|
|||||||
|
|
||||||
db = createDb(config.databaseUrl);
|
db = createDb(config.databaseUrl);
|
||||||
logger.info("Using external PostgreSQL via DATABASE_URL/config");
|
logger.info("Using external PostgreSQL via DATABASE_URL/config");
|
||||||
|
activeDatabaseConnectionString = config.databaseUrl;
|
||||||
startupDbInfo = { mode: "external-postgres", connectionString: config.databaseUrl };
|
startupDbInfo = { mode: "external-postgres", connectionString: config.databaseUrl };
|
||||||
} else {
|
} else {
|
||||||
const moduleName = "embedded-postgres";
|
const moduleName = "embedded-postgres";
|
||||||
@@ -364,6 +368,7 @@ if (config.databaseUrl) {
|
|||||||
|
|
||||||
db = createDb(embeddedConnectionString);
|
db = createDb(embeddedConnectionString);
|
||||||
logger.info("Embedded PostgreSQL ready");
|
logger.info("Embedded PostgreSQL ready");
|
||||||
|
activeDatabaseConnectionString = embeddedConnectionString;
|
||||||
startupDbInfo = { mode: "embedded-postgres", dataDir, port };
|
startupDbInfo = { mode: "embedded-postgres", dataDir, port };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -489,6 +494,54 @@ if (config.heartbeatSchedulerEnabled) {
|
|||||||
}, config.heartbeatSchedulerIntervalMs);
|
}, config.heartbeatSchedulerIntervalMs);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (config.databaseBackupEnabled) {
|
||||||
|
const backupIntervalMs = config.databaseBackupIntervalMinutes * 60 * 1000;
|
||||||
|
let backupInFlight = false;
|
||||||
|
|
||||||
|
const runScheduledBackup = async () => {
|
||||||
|
if (backupInFlight) {
|
||||||
|
logger.warn("Skipping scheduled database backup because a previous backup is still running");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
backupInFlight = true;
|
||||||
|
try {
|
||||||
|
const result = await runDatabaseBackup({
|
||||||
|
connectionString: activeDatabaseConnectionString,
|
||||||
|
backupDir: config.databaseBackupDir,
|
||||||
|
retentionDays: config.databaseBackupRetentionDays,
|
||||||
|
filenamePrefix: "paperclip",
|
||||||
|
});
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
backupFile: result.backupFile,
|
||||||
|
sizeBytes: result.sizeBytes,
|
||||||
|
prunedCount: result.prunedCount,
|
||||||
|
backupDir: config.databaseBackupDir,
|
||||||
|
retentionDays: config.databaseBackupRetentionDays,
|
||||||
|
},
|
||||||
|
`Automatic database backup complete: ${formatDatabaseBackupResult(result)}`,
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
logger.error({ err, backupDir: config.databaseBackupDir }, "Automatic database backup failed");
|
||||||
|
} finally {
|
||||||
|
backupInFlight = false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
intervalMinutes: config.databaseBackupIntervalMinutes,
|
||||||
|
retentionDays: config.databaseBackupRetentionDays,
|
||||||
|
backupDir: config.databaseBackupDir,
|
||||||
|
},
|
||||||
|
"Automatic database backups enabled",
|
||||||
|
);
|
||||||
|
setInterval(() => {
|
||||||
|
void runScheduledBackup();
|
||||||
|
}, backupIntervalMs);
|
||||||
|
}
|
||||||
|
|
||||||
server.listen(listenPort, config.host, () => {
|
server.listen(listenPort, config.host, () => {
|
||||||
logger.info(`Server listening on ${config.host}:${listenPort}`);
|
logger.info(`Server listening on ${config.host}:${listenPort}`);
|
||||||
if (process.env.PAPERCLIP_OPEN_ON_LISTEN === "true") {
|
if (process.env.PAPERCLIP_OPEN_ON_LISTEN === "true") {
|
||||||
@@ -515,6 +568,10 @@ server.listen(listenPort, config.host, () => {
|
|||||||
migrationSummary,
|
migrationSummary,
|
||||||
heartbeatSchedulerEnabled: config.heartbeatSchedulerEnabled,
|
heartbeatSchedulerEnabled: config.heartbeatSchedulerEnabled,
|
||||||
heartbeatSchedulerIntervalMs: config.heartbeatSchedulerIntervalMs,
|
heartbeatSchedulerIntervalMs: config.heartbeatSchedulerIntervalMs,
|
||||||
|
databaseBackupEnabled: config.databaseBackupEnabled,
|
||||||
|
databaseBackupIntervalMinutes: config.databaseBackupIntervalMinutes,
|
||||||
|
databaseBackupRetentionDays: config.databaseBackupRetentionDays,
|
||||||
|
databaseBackupDir: config.databaseBackupDir,
|
||||||
});
|
});
|
||||||
|
|
||||||
const boardClaimUrl = getBoardClaimWarningUrl(config.host, listenPort);
|
const boardClaimUrl = getBoardClaimWarningUrl(config.host, listenPort);
|
||||||
|
|||||||
@@ -29,6 +29,10 @@ type StartupBannerOptions = {
|
|||||||
migrationSummary: string;
|
migrationSummary: string;
|
||||||
heartbeatSchedulerEnabled: boolean;
|
heartbeatSchedulerEnabled: boolean;
|
||||||
heartbeatSchedulerIntervalMs: number;
|
heartbeatSchedulerIntervalMs: number;
|
||||||
|
databaseBackupEnabled: boolean;
|
||||||
|
databaseBackupIntervalMinutes: number;
|
||||||
|
databaseBackupRetentionDays: number;
|
||||||
|
databaseBackupDir: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
const ansi = {
|
const ansi = {
|
||||||
@@ -125,6 +129,9 @@ export function printStartupBanner(opts: StartupBannerOptions): void {
|
|||||||
const heartbeat = opts.heartbeatSchedulerEnabled
|
const heartbeat = opts.heartbeatSchedulerEnabled
|
||||||
? `enabled ${color(`(${opts.heartbeatSchedulerIntervalMs}ms)`, "dim")}`
|
? `enabled ${color(`(${opts.heartbeatSchedulerIntervalMs}ms)`, "dim")}`
|
||||||
: color("disabled", "yellow");
|
: color("disabled", "yellow");
|
||||||
|
const dbBackup = opts.databaseBackupEnabled
|
||||||
|
? `enabled ${color(`(every ${opts.databaseBackupIntervalMinutes}m, keep ${opts.databaseBackupRetentionDays}d)`, "dim")}`
|
||||||
|
: color("disabled", "yellow");
|
||||||
|
|
||||||
const art = [
|
const art = [
|
||||||
color("██████╗ █████╗ ██████╗ ███████╗██████╗ ██████╗██╗ ██╗██████╗ ", "cyan"),
|
color("██████╗ █████╗ ██████╗ ███████╗██████╗ ██████╗██╗ ██╗██████╗ ", "cyan"),
|
||||||
@@ -154,6 +161,8 @@ export function printStartupBanner(opts: StartupBannerOptions): void {
|
|||||||
: color(agentJwtSecret.message, "yellow"),
|
: color(agentJwtSecret.message, "yellow"),
|
||||||
),
|
),
|
||||||
row("Heartbeat", heartbeat),
|
row("Heartbeat", heartbeat),
|
||||||
|
row("DB Backup", dbBackup),
|
||||||
|
row("Backup Dir", opts.databaseBackupDir),
|
||||||
row("Config", configPath),
|
row("Config", configPath),
|
||||||
agentJwtSecret.status === "warn"
|
agentJwtSecret.status === "warn"
|
||||||
? color(" ───────────────────────────────────────────────────────", "yellow")
|
? color(" ───────────────────────────────────────────────────────", "yellow")
|
||||||
|
|||||||
Reference in New Issue
Block a user