eliott/paperclip
1
0
Fork 0
mirror of https://github.com/paperclipai/paperclip synced 2026-04-25 17:25:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(backups): tiered daily/weekly/monthly retention with UI controls

Browse Source 
Replace single retentionDays with a three-tier BackupRetentionPolicy:
- Daily: keep all backups (presets: 3, 7, 14 days; default 7)
- Weekly: keep one per calendar week (presets: 1, 2, 4 weeks; default 4)
- Monthly: keep one per calendar month (presets: 1, 3, 6 months; default 1)

Pruning sorts backups newest-first and applies each tier's cutoff,
keeping only the newest entry per ISO week/month bucket. The Instance
Settings General page now shows three preset selectors (no icon, matches
existing page design). Remove Database icon import.
This commit is contained in:
Aron Prins
2026-04-07 09:54:39 +02:00
parent cc44d309c0
commit fcbae62baf
13 changed files with 243 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cli/src/commands/db-backup.ts
View File

@@ -73,7 +73,7 @@ export async function dbBackupCommand(opts: DbBackupOptions): Promise<void> {
const result = await runDatabaseBackup({
connectionString: connection.value,
backupDir,
retentionDays,
retention: { dailyDays: retentionDays, weeklyWeeks: 4, monthlyMonths: 1 },
filenamePrefix,
});
spinner.stop(`Backup saved: ${formatDatabaseBackupResult(result)}`);

2
cli/src/commands/worktree.ts
View File

@@ -903,7 +903,7 @@ async function seedWorktreeDatabase(input: {
const backup = await runDatabaseBackup({
connectionString: sourceConnectionString,
backupDir: path.resolve(input.targetPaths.backupDir, "seed"),
retentionDays: 7,
retention: { dailyDays: 7, weeklyWeeks: 4, monthlyMonths: 1 },
filenamePrefix: `${input.instanceId}-seed`,
includeMigrationJournal: true,
excludeTables: seedPlan.excludedTables,

2
packages/db/src/backup-lib.test.ts
View File

@@ -125,7 +125,7 @@ describeEmbeddedPostgres("runDatabaseBackup", () => {
const result = await runDatabaseBackup({
connectionString: sourceConnectionString,
backupDir,
retentionDays: 7,
retention: { dailyDays: 7, weeklyWeeks: 4, monthlyMonths: 1 },
filenamePrefix: "paperclip-test",
});

97
packages/db/src/backup-lib.ts
View File

@@ -5,10 +5,16 @@ import { pipeline } from "node:stream/promises";
import { createGunzip, createGzip } from "node:zlib";
import postgres from "postgres";
export type BackupRetentionPolicy = {
dailyDays: number;
weeklyWeeks: number;
monthlyMonths: number;
};
export type RunDatabaseBackupOptions = {
connectionString: string;
backupDir: string;
retentionDays: number;
retention: BackupRetentionPolicy;
filenamePrefix?: string;
connectTimeoutSeconds?: number;
includeMigrationJournal?: boolean;
@@ -77,24 +83,91 @@ function timestamp(date: Date = new Date()): string {
return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}-${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`;
}
function pruneOldBackups(backupDir: string, retentionDays: number, filenamePrefix: string): number {
/**
* ISO week key for grouping backups by calendar week (ISO 8601).
*/
function isoWeekKey(date: Date): string {
const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()));
d.setUTCDate(d.getUTCDate() + 4 - (d.getUTCDay() || 7));
const yearStart = new Date(Date.UTC(d.getUTCFullYear(), 0, 1));
const weekNo = Math.ceil(((d.getTime() - yearStart.getTime()) / 86400000 + 1) / 7);
return `${d.getUTCFullYear()}-W${String(weekNo).padStart(2, "0")}`;
}
function monthKey(date: Date): string {
return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}`;
}
/**
* Tiered backup pruning:
* - Daily tier: keep ALL backups from the last `dailyDays` days
* - Weekly tier: keep the NEWEST backup per calendar week for `weeklyWeeks` weeks
* - Monthly tier: keep the NEWEST backup per calendar month for `monthlyMonths` months
* - Everything else is deleted
*/
function pruneOldBackups(backupDir: string, retention: BackupRetentionPolicy, filenamePrefix: string): number {
if (!existsSync(backupDir)) return 0;
const safeRetention = Math.max(1, Math.trunc(retentionDays));
const cutoff = Date.now() - safeRetention * 24 * 60 * 60 * 1000;
let pruned = 0;
const now = Date.now();
const dailyCutoff = now - Math.max(1, retention.dailyDays) * 24 * 60 * 60 * 1000;
const weeklyCutoff = now - Math.max(1, retention.weeklyWeeks) * 7 * 24 * 60 * 60 * 1000;
const monthlyCutoff = now - Math.max(1, retention.monthlyMonths) * 30 * 24 * 60 * 60 * 1000;
type BackupEntry = { name: string; fullPath: string; mtimeMs: number };
const entries: BackupEntry[] = [];
for (const name of readdirSync(backupDir)) {
if (!name.startsWith(`${filenamePrefix}-`)) continue;
if (!name.endsWith(".sql") && !name.endsWith(".sql.gz")) continue;
const fullPath = resolve(backupDir, name);
const stat = statSync(fullPath);
if (stat.mtimeMs < cutoff) {
unlinkSync(fullPath);
pruned++;
}
entries.push({ name, fullPath, mtimeMs: stat.mtimeMs });
}
return pruned;
// Sort newest first so the first entry per week/month bucket is the one we keep
entries.sort((a, b) => b.mtimeMs - a.mtimeMs);
const keepWeekBuckets = new Set<string>();
const keepMonthBuckets = new Set<string>();
const toDelete: string[] = [];
for (const entry of entries) {
// Daily tier — keep everything within dailyDays
if (entry.mtimeMs >= dailyCutoff) continue;
const date = new Date(entry.mtimeMs);
const week = isoWeekKey(date);
const month = monthKey(date);
// Weekly tier — keep newest per calendar week
if (entry.mtimeMs >= weeklyCutoff) {
if (keepWeekBuckets.has(week)) {
toDelete.push(entry.fullPath);
} else {
keepWeekBuckets.add(week);
}
continue;
}
// Monthly tier — keep newest per calendar month
if (entry.mtimeMs >= monthlyCutoff) {
if (keepMonthBuckets.has(month)) {
toDelete.push(entry.fullPath);
} else {
keepMonthBuckets.add(month);
}
continue;
}
// Beyond all retention tiers — delete
toDelete.push(entry.fullPath);
}
for (const filePath of toDelete) {
unlinkSync(filePath);
}
return toDelete.length;
}
function formatBackupSize(sizeBytes: number): string {
@@ -287,7 +360,7 @@ export function createBufferedTextFileWriter(filePath: string, maxBufferedBytes
export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise<RunDatabaseBackupResult> {
const filenamePrefix = opts.filenamePrefix ?? "paperclip";
const retentionDays = Math.max(1, Math.trunc(opts.retentionDays));
const retention = opts.retention;
const connectTimeout = Math.max(1, Math.trunc(opts.connectTimeoutSeconds ?? 5));
const includeMigrationJournal = opts.includeMigrationJournal === true;
const excludedTableNames = normalizeTableNameSet(opts.excludeTables);
@@ -678,7 +751,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
unlinkSync(sqlFile);
const sizeBytes = statSync(backupFile).size;
const prunedCount = pruneOldBackups(opts.backupDir, retentionDays, filenamePrefix);
const prunedCount = pruneOldBackups(opts.backupDir, retention, filenamePrefix);
return {
backupFile,

4
packages/db/src/backup.ts
View File

@@ -85,7 +85,7 @@ function resolveBackupDir(config: PartialConfig | null): string {
}
function resolveRetentionDays(config: PartialConfig | null): number {
return asPositiveInt(config?.database?.backup?.retentionDays) ?? 30;
return asPositiveInt(config?.database?.backup?.retentionDays) ?? 7;
}
async function main() {
@@ -103,7 +103,7 @@ async function main() {
const result = await runDatabaseBackup({
connectionString,
backupDir,
retentionDays,
retention: { dailyDays: retentionDays, weeklyWeeks: 4, monthlyMonths: 1 },
filenamePrefix: "paperclip",
});

1
packages/db/src/index.ts
View File

@@ -21,6 +21,7 @@ export {
runDatabaseBackup,
runDatabaseRestore,
formatDatabaseBackupResult,
type BackupRetentionPolicy,
type RunDatabaseBackupOptions,
type RunDatabaseBackupResult,
type RunDatabaseRestoreOptions,

8
packages/shared/src/index.ts
View File

@@ -189,7 +189,7 @@ export type {
InstanceExperimentalSettings,
InstanceGeneralSettings,
InstanceSettings,
BackupRetentionDays,
BackupRetentionPolicy,
Agent,
AgentAccessState,
AgentChainOfCommandEntry,
@@ -371,8 +371,10 @@ export {
} from "./types/feedback.js";
export {
BACKUP_RETENTION_PRESETS,
DEFAULT_BACKUP_RETENTION_DAYS,
DAILY_RETENTION_PRESETS,
WEEKLY_RETENTION_PRESETS,
MONTHLY_RETENTION_PRESETS,
DEFAULT_BACKUP_RETENTION,
} from "./types/instance.js";
export {

4
packages/shared/src/types/index.ts
View File

@@ -11,8 +11,8 @@ export type {
FeedbackTraceBundleFile,
FeedbackTraceBundle,
} from "./feedback.js";
export type { InstanceExperimentalSettings, InstanceGeneralSettings, InstanceSettings, BackupRetentionDays } from "./instance.js";
export { BACKUP_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION_DAYS } from "./instance.js";
export type { InstanceExperimentalSettings, InstanceGeneralSettings, InstanceSettings, BackupRetentionPolicy } from "./instance.js";
export { DAILY_RETENTION_PRESETS, WEEKLY_RETENTION_PRESETS, MONTHLY_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION } from "./instance.js";
export type {
CompanySkillSourceType,
CompanySkillTrustLevel,

20
packages/shared/src/types/instance.ts
View File

@@ -1,14 +1,26 @@
import type { FeedbackDataSharingPreference } from "./feedback.js";
export const BACKUP_RETENTION_PRESETS = [7, 14, 30] as const;
export type BackupRetentionDays = (typeof BACKUP_RETENTION_PRESETS)[number];
export const DEFAULT_BACKUP_RETENTION_DAYS: BackupRetentionDays = 7;
export const DAILY_RETENTION_PRESETS = [3, 7, 14] as const;
export const WEEKLY_RETENTION_PRESETS = [1, 2, 4] as const;
export const MONTHLY_RETENTION_PRESETS = [1, 3, 6] as const;
export interface BackupRetentionPolicy {
dailyDays: (typeof DAILY_RETENTION_PRESETS)[number];
weeklyWeeks: (typeof WEEKLY_RETENTION_PRESETS)[number];
monthlyMonths: (typeof MONTHLY_RETENTION_PRESETS)[number];
}
export const DEFAULT_BACKUP_RETENTION: BackupRetentionPolicy = {
dailyDays: 7,
weeklyWeeks: 4,
monthlyMonths: 1,
};
export interface InstanceGeneralSettings {
censorUsernameInLogs: boolean;
keyboardShortcuts: boolean;
feedbackDataSharingPreference: FeedbackDataSharingPreference;
backupRetentionDays: BackupRetentionDays;
backupRetention: BackupRetentionPolicy;
}
export interface InstanceExperimentalSettings {

26
packages/shared/src/validators/instance.ts
View File

@@ -1,13 +1,25 @@
import { z } from "zod";
import { DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE } from "../types/feedback.js";
import { BACKUP_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION_DAYS } from "../types/instance.js";
import {
DAILY_RETENTION_PRESETS,
WEEKLY_RETENTION_PRESETS,
MONTHLY_RETENTION_PRESETS,
DEFAULT_BACKUP_RETENTION,
} from "../types/instance.js";
import { feedbackDataSharingPreferenceSchema } from "./feedback.js";
export const backupRetentionDaysSchema = z.number().refine(
(v): v is (typeof BACKUP_RETENTION_PRESETS)[number] =>
(BACKUP_RETENTION_PRESETS as readonly number[]).includes(v),
{ message: `Must be one of: ${BACKUP_RETENTION_PRESETS.join(", ")}` },
);
function presetSchema<T extends readonly number[]>(presets: T, label: string) {
return z.number().refine(
(v): v is T[number] => (presets as readonly number[]).includes(v),
{ message: `${label} must be one of: ${presets.join(", ")}` },
);
}
export const backupRetentionPolicySchema = z.object({
dailyDays: presetSchema(DAILY_RETENTION_PRESETS, "dailyDays").default(DEFAULT_BACKUP_RETENTION.dailyDays),
weeklyWeeks: presetSchema(WEEKLY_RETENTION_PRESETS, "weeklyWeeks").default(DEFAULT_BACKUP_RETENTION.weeklyWeeks),
monthlyMonths: presetSchema(MONTHLY_RETENTION_PRESETS, "monthlyMonths").default(DEFAULT_BACKUP_RETENTION.monthlyMonths),
});
export const instanceGeneralSettingsSchema = z.object({
censorUsernameInLogs: z.boolean().default(false),
@@ -15,7 +27,7 @@ export const instanceGeneralSettingsSchema = z.object({
feedbackDataSharingPreference: feedbackDataSharingPreferenceSchema.default(
DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE,
),
backupRetentionDays: backupRetentionDaysSchema.default(DEFAULT_BACKUP_RETENTION_DAYS),
backupRetention: backupRetentionPolicySchema.default(DEFAULT_BACKUP_RETENTION),
}).strict();
export const patchInstanceGeneralSettingsSchema = instanceGeneralSettingsSchema.partial();

6
server/src/index.ts
View File

@@ -642,12 +642,12 @@ export async function startServer(): Promise<StartedServer> {
try {
// Read retention from Instance Settings (DB) so changes take effect without restart
const generalSettings = await settingsSvc.getGeneral();
const retentionDays = generalSettings.backupRetentionDays;
const retention = generalSettings.backupRetention;
const result = await runDatabaseBackup({
connectionString: activeDatabaseConnectionString,
backupDir: config.databaseBackupDir,
retentionDays,
retention,
filenamePrefix: "paperclip",
});
logger.info(
@@ -656,7 +656,7 @@ export async function startServer(): Promise<StartedServer> {
sizeBytes: result.sizeBytes,
prunedCount: result.prunedCount,
backupDir: config.databaseBackupDir,
retentionDays,
retention,
},
`Automatic database backup complete: ${formatDatabaseBackupResult(result)}`,
);

6
server/src/services/instance-settings.ts
View File

@@ -2,7 +2,7 @@ import type { Db } from "@paperclipai/db";
import { companies, instanceSettings } from "@paperclipai/db";
import {
DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE,
DEFAULT_BACKUP_RETENTION_DAYS,
DEFAULT_BACKUP_RETENTION,
instanceGeneralSettingsSchema,
type InstanceGeneralSettings,
instanceExperimentalSettingsSchema,
@@ -23,14 +23,14 @@ function normalizeGeneralSettings(raw: unknown): InstanceGeneralSettings {
keyboardShortcuts: parsed.data.keyboardShortcuts ?? false,
feedbackDataSharingPreference:
parsed.data.feedbackDataSharingPreference ?? DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE,
backupRetentionDays: parsed.data.backupRetentionDays ?? DEFAULT_BACKUP_RETENTION_DAYS,
backupRetention: parsed.data.backupRetention ?? DEFAULT_BACKUP_RETENTION,
};
}
return {
censorUsernameInLogs: false,
keyboardShortcuts: false,
feedbackDataSharingPreference: DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE,
backupRetentionDays: DEFAULT_BACKUP_RETENTION_DAYS,
backupRetention: DEFAULT_BACKUP_RETENTION,
};
}

144
ui/src/pages/InstanceGeneralSettings.tsx
View File

@@ -1,8 +1,13 @@
import { useEffect, useState } from "react";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import type { PatchInstanceGeneralSettings, BackupRetentionDays } from "@paperclipai/shared";
import { BACKUP_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION_DAYS } from "@paperclipai/shared";
import { Database, LogOut, SlidersHorizontal } from "lucide-react";
import type { PatchInstanceGeneralSettings, BackupRetentionPolicy } from "@paperclipai/shared";
import {
DAILY_RETENTION_PRESETS,
WEEKLY_RETENTION_PRESETS,
MONTHLY_RETENTION_PRESETS,
DEFAULT_BACKUP_RETENTION,
} from "@paperclipai/shared";
import { LogOut, SlidersHorizontal } from "lucide-react";
import { authApi } from "@/api/auth";
import { instanceSettingsApi } from "@/api/instanceSettings";
import { Button } from "../components/ui/button";
@@ -68,7 +73,7 @@ export function InstanceGeneralSettings() {
const censorUsernameInLogs = generalQuery.data?.censorUsernameInLogs === true;
const keyboardShortcuts = generalQuery.data?.keyboardShortcuts === true;
const feedbackDataSharingPreference = generalQuery.data?.feedbackDataSharingPreference ?? "prompt";
const backupRetentionDays: BackupRetentionDays = generalQuery.data?.backupRetentionDays ?? DEFAULT_BACKUP_RETENTION_DAYS;
const backupRetention: BackupRetentionPolicy = generalQuery.data?.backupRetention ?? DEFAULT_BACKUP_RETENTION;
return (
<div className="max-w-4xl space-y-6">
@@ -126,44 +131,103 @@ export function InstanceGeneralSettings() {
</section>
<section className="rounded-xl border border-border bg-card p-5">
<div className="space-y-4">
<div className="flex items-center gap-2">
<Database className="h-4 w-4 text-muted-foreground" />
<div className="space-y-1.5">
<h2 className="text-sm font-semibold">Backup retention</h2>
<p className="max-w-2xl text-sm text-muted-foreground">
How long to keep automatic database backups before pruning. Backups are compressed
with gzip to minimize disk usage.
</p>
<div className="space-y-5">
<div className="space-y-1.5">
<h2 className="text-sm font-semibold">Backup retention</h2>
<p className="max-w-2xl text-sm text-muted-foreground">
Configure how long to keep automatic database backups at each tier. Daily backups
are kept in full, then thinned to one per week and one per month. Backups are
compressed with gzip.
</p>
</div>
<div className="space-y-1.5">
<h3 className="text-xs font-medium text-muted-foreground uppercase tracking-wide">Daily</h3>
<div className="flex flex-wrap gap-2">
{DAILY_RETENTION_PRESETS.map((days) => {
const active = backupRetention.dailyDays === days;
return (
<button
key={days}
type="button"
disabled={updateGeneralMutation.isPending}
className={cn(
"rounded-lg border px-3 py-2 text-left transition-colors disabled:cursor-not-allowed disabled:opacity-60",
active
? "border-foreground bg-accent text-foreground"
: "border-border bg-background hover:bg-accent/50",
)}
onClick={() =>
updateGeneralMutation.mutate({
backupRetention: { ...backupRetention, dailyDays: days },
})
}
>
<div className="text-sm font-medium">{days} days</div>
</button>
);
})}
</div>
</div>
<div className="flex flex-wrap gap-2">
{BACKUP_RETENTION_PRESETS.map((days) => {
const active = backupRetentionDays === days;
const label =
days === 7 ? "7 days" : days === 14 ? "2 weeks" : "1 month";
return (
<button
key={days}
type="button"
disabled={updateGeneralMutation.isPending}
className={cn(
"rounded-lg border px-3 py-2 text-left transition-colors disabled:cursor-not-allowed disabled:opacity-60",
active
? "border-foreground bg-accent text-foreground"
: "border-border bg-background hover:bg-accent/50",
)}
onClick={() =>
updateGeneralMutation.mutate({ backupRetentionDays: days })
}
>
<div className="text-sm font-medium">{label}</div>
<div className="text-xs text-muted-foreground">
Keep backups for {days} days
</div>
</button>
);
})}
<div className="space-y-1.5">
<h3 className="text-xs font-medium text-muted-foreground uppercase tracking-wide">Weekly</h3>
<div className="flex flex-wrap gap-2">
{WEEKLY_RETENTION_PRESETS.map((weeks) => {
const active = backupRetention.weeklyWeeks === weeks;
const label = weeks === 1 ? "1 week" : `${weeks} weeks`;
return (
<button
key={weeks}
type="button"
disabled={updateGeneralMutation.isPending}
className={cn(
"rounded-lg border px-3 py-2 text-left transition-colors disabled:cursor-not-allowed disabled:opacity-60",
active
? "border-foreground bg-accent text-foreground"
: "border-border bg-background hover:bg-accent/50",
)}
onClick={() =>
updateGeneralMutation.mutate({
backupRetention: { ...backupRetention, weeklyWeeks: weeks },
})
}
>
<div className="text-sm font-medium">{label}</div>
</button>
);
})}
</div>
</div>
<div className="space-y-1.5">
<h3 className="text-xs font-medium text-muted-foreground uppercase tracking-wide">Monthly</h3>
<div className="flex flex-wrap gap-2">
{MONTHLY_RETENTION_PRESETS.map((months) => {
const active = backupRetention.monthlyMonths === months;
const label = months === 1 ? "1 month" : `${months} months`;
return (
<button
key={months}
type="button"
disabled={updateGeneralMutation.isPending}
className={cn(
"rounded-lg border px-3 py-2 text-left transition-colors disabled:cursor-not-allowed disabled:opacity-60",
active
? "border-foreground bg-accent text-foreground"
: "border-border bg-background hover:bg-accent/50",
)}
onClick={() =>
updateGeneralMutation.mutate({
backupRetention: { ...backupRetention, monthlyMonths: months },
})
}
>
<div className="text-sm font-medium">{label}</div>
</button>
);
})}
</div>
</div>
</div>
</section>