diff --git a/docs/docs.json b/docs/docs.json index 90789e06bb..f87809af07 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -46,6 +46,7 @@ "guides/board-operator/managing-agents", "guides/board-operator/org-structure", "guides/board-operator/managing-tasks", + "guides/board-operator/execution-workspaces-and-runtime-services", "guides/board-operator/delegation", "guides/board-operator/approvals", "guides/board-operator/costs-and-budgets", diff --git a/docs/guides/board-operator/execution-workspaces-and-runtime-services.md b/docs/guides/board-operator/execution-workspaces-and-runtime-services.md new file mode 100644 index 0000000000..285d701a48 --- /dev/null +++ b/docs/guides/board-operator/execution-workspaces-and-runtime-services.md @@ -0,0 +1,68 @@ +--- +title: Execution Workspaces And Runtime Services +summary: How project runtime configuration, execution workspaces, and issue runs fit together +--- + +This guide documents the intended runtime model for projects, execution workspaces, and issue runs in Paperclip. + +## Project runtime configuration + +You can define how to run a project on the project workspace itself. + +- Project workspace runtime config describes how to run services for that project checkout. +- This is the default runtime configuration that child execution workspaces may inherit. +- Defining the config does not start anything by itself. + +## Manual runtime control + +Runtime services are manually controlled from the UI. + +- Project workspace runtime services are started and stopped from the project workspace UI. +- Execution workspace runtime services are started and stopped from the execution workspace UI. +- Paperclip does not automatically start or stop these runtime services as part of issue execution. +- Paperclip also does not automatically restart workspace runtime services on server boot. + +## Execution workspace inheritance + +Execution workspaces isolate code and runtime state from the project primary workspace. + +- An isolated execution workspace has its own checkout path, branch, and local runtime instance. +- The runtime configuration may inherit from the linked project workspace by default. +- The execution workspace may override that runtime configuration with its own workspace-specific settings. +- The inherited configuration answers "how to run the service", but the running process is still specific to that execution workspace. + +## Issues and execution workspaces + +Issues are attached to execution workspace behavior, not to automatic runtime management. + +- An issue may create a new execution workspace when you choose an isolated workspace mode. +- An issue may reuse an existing execution workspace when you choose reuse. +- Multiple issues may intentionally share one execution workspace so they can work against the same branch and running runtime services. +- Assigning or running an issue does not automatically start or stop runtime services for that workspace. + +## Execution workspace lifecycle + +Execution workspaces are durable until a human closes them. + +- The UI can archive an execution workspace. +- Closing an execution workspace stops its runtime services and cleans up its workspace artifacts when allowed. +- Shared workspaces that point at the project primary checkout are treated more conservatively during cleanup than disposable isolated workspaces. + +## Resolved workspace logic during heartbeat runs + +Heartbeat still resolves a workspace for the run, but that is about code location and session continuity, not runtime-service control. + +1. Heartbeat resolves a base workspace for the run. +2. Paperclip realizes the effective execution workspace, including creating or reusing a worktree when needed. +3. Paperclip persists execution-workspace metadata such as paths, refs, and provisioning settings. +4. Heartbeat passes the resolved code workspace to the agent run. +5. Workspace runtime services remain manual UI-managed controls rather than automatic heartbeat-managed services. + +## Current implementation guarantees + +With the current implementation: + +- Project workspace runtime config is the fallback for execution workspace UI controls. +- Execution workspace runtime overrides are stored on the execution workspace. +- Heartbeat runs do not auto-start workspace runtime services. +- Server startup does not auto-restart workspace runtime services. diff --git a/packages/adapters/claude-local/src/index.ts b/packages/adapters/claude-local/src/index.ts index b28ae18097..41c0693f8d 100644 --- a/packages/adapters/claude-local/src/index.ts +++ b/packages/adapters/claude-local/src/index.ts @@ -26,7 +26,7 @@ Core fields: - extraArgs (string[], optional): additional CLI args - env (object, optional): KEY=VALUE environment variables - workspaceStrategy (object, optional): execution workspace strategy; currently supports { type: "git_worktree", baseRef?, branchTemplate?, worktreeParentDir? } -- workspaceRuntime (object, optional): workspace runtime service intents; local host-managed services are realized before Claude starts and exposed back via context/env +- workspaceRuntime (object, optional): reserved for workspace runtime metadata; workspace runtime services are manually controlled from the workspace UI and are not auto-started by heartbeats Operational fields: - timeoutSec (number, optional): run timeout in seconds diff --git a/packages/adapters/codex-local/src/index.ts b/packages/adapters/codex-local/src/index.ts index 10cf6fe974..afb2cd9b72 100644 --- a/packages/adapters/codex-local/src/index.ts +++ b/packages/adapters/codex-local/src/index.ts @@ -32,7 +32,7 @@ Core fields: - extraArgs (string[], optional): additional CLI args - env (object, optional): KEY=VALUE environment variables - workspaceStrategy (object, optional): execution workspace strategy; currently supports { type: "git_worktree", baseRef?, branchTemplate?, worktreeParentDir? } -- workspaceRuntime (object, optional): workspace runtime service intents; local host-managed services are realized before Codex starts and exposed back via context/env +- workspaceRuntime (object, optional): reserved for workspace runtime metadata; workspace runtime services are manually controlled from the workspace UI and are not auto-started by heartbeats Operational fields: - timeoutSec (number, optional): run timeout in seconds diff --git a/packages/adapters/openclaw-gateway/src/index.ts b/packages/adapters/openclaw-gateway/src/index.ts index 195edfbf8d..1bdf66f2e6 100644 --- a/packages/adapters/openclaw-gateway/src/index.ts +++ b/packages/adapters/openclaw-gateway/src/index.ts @@ -31,7 +31,7 @@ Gateway connect identity fields: Request behavior fields: - payloadTemplate (object, optional): additional fields merged into gateway agent params -- workspaceRuntime (object, optional): desired runtime service intents; Paperclip forwards these in a standardized paperclip.workspaceRuntime block for remote execution environments +- workspaceRuntime (object, optional): reserved workspace runtime metadata; workspace runtime services are manually controlled from the workspace UI and are not auto-started by heartbeats - timeoutSec (number, optional): adapter timeout in seconds (default 120) - waitTimeoutMs (number, optional): agent.wait timeout override (default timeoutSec * 1000) - autoPairOnFirstConnect (boolean, optional): on first "pairing required", attempt device.pair.list/device.pair.approve via shared auth, then retry once (default true) @@ -45,7 +45,7 @@ Standard outbound payload additions: - paperclip (object): standardized Paperclip context added to every gateway agent request - paperclip.workspace (object, optional): resolved execution workspace for this run - paperclip.workspaces (array, optional): additional workspace hints Paperclip exposed to the run -- paperclip.workspaceRuntime (object, optional): normalized runtime service intent config for the workspace +- paperclip.workspaceRuntime (object, optional): reserved workspace runtime metadata when explicitly supplied outside normal heartbeat execution Standard result metadata supported: - meta.runtimeServices (array, optional): normalized adapter-managed runtime service reports diff --git a/scripts/kill-dev.sh b/scripts/kill-dev.sh index 2cb946e29b..9a53498a4f 100755 --- a/scripts/kill-dev.sh +++ b/scripts/kill-dev.sh @@ -8,64 +8,199 @@ # set -euo pipefail +shopt -s nullglob DRY_RUN=false if [[ "${1:-}" == "--dry" || "${1:-}" == "--dry-run" || "${1:-}" == "-n" ]]; then DRY_RUN=true fi -# Collect PIDs of node processes running from any paperclip directory. -# Matches paths like /Users/*/paperclip/... or /Users/*/paperclip-*/... -# Excludes postgres-related processes. -pids=() -lines=() +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +REPO_PARENT="$(dirname "$REPO_ROOT")" + +node_pids=() +node_lines=() +pg_pids=() +pg_pidfiles=() +pg_data_dirs=() + +is_pid_running() { + local pid="$1" + kill -0 "$pid" 2>/dev/null +} + +read_pidfile_pid() { + local pidfile="$1" + local first_line + first_line="$(head -n 1 "$pidfile" 2>/dev/null | tr -d '[:space:]' || true)" + if [[ "$first_line" =~ ^[0-9]+$ ]] && (( first_line > 0 )); then + printf '%s\n' "$first_line" + return 0 + fi + return 1 +} + +command_for_pid() { + local pid="$1" + ps -o command= -p "$pid" 2>/dev/null || true +} + +append_postgres_from_pidfile() { + local pidfile="$1" + local pid cmd data_dir + pid="$(read_pidfile_pid "$pidfile" || true)" + [[ -n "$pid" ]] || return 0 + is_pid_running "$pid" || return 0 + cmd="$(command_for_pid "$pid")" + [[ "$cmd" == *postgres* ]] || return 0 + + for existing_pid in "${pg_pids[@]:-}"; do + [[ "$existing_pid" == "$pid" ]] && return 0 + done + + data_dir="$(dirname "$pidfile")" + pg_pids+=("$pid") + pg_pidfiles+=("$pidfile") + pg_data_dirs+=("$data_dir") +} + +wait_for_pid_exit() { + local pid="$1" + local timeout_sec="$2" + local waited=0 + while is_pid_running "$pid"; do + if (( waited >= timeout_sec * 10 )); then + return 1 + fi + sleep 0.1 + ((waited += 1)) + done + return 0 +} while IFS= read -r line; do [[ -z "$line" ]] && continue - # skip postgres processes [[ "$line" == *postgres* ]] && continue pid=$(echo "$line" | awk '{print $2}') - pids+=("$pid") - lines+=("$line") + node_pids+=("$pid") + node_lines+=("$line") done < <(ps aux | grep -E '/paperclip(-[^/]+)?/' | grep node | grep -v grep || true) -if [[ ${#pids[@]} -eq 0 ]]; then +candidate_pidfiles=() +candidate_pidfiles+=( + "$HOME"/.paperclip/instances/*/db/postmaster.pid + "$REPO_ROOT"/.paperclip/instances/*/db/postmaster.pid + "$REPO_ROOT"/.paperclip/runtime-services/instances/*/db/postmaster.pid +) + +for sibling_root in "$REPO_PARENT"/paperclip*; do + [[ -d "$sibling_root" ]] || continue + candidate_pidfiles+=( + "$sibling_root"/.paperclip/instances/*/db/postmaster.pid + "$sibling_root"/.paperclip/runtime-services/instances/*/db/postmaster.pid + ) +done + +for pidfile in "${candidate_pidfiles[@]:-}"; do + [[ -f "$pidfile" ]] || continue + append_postgres_from_pidfile "$pidfile" +done + +if [[ ${#node_pids[@]} -eq 0 && ${#pg_pids[@]} -eq 0 ]]; then echo "No Paperclip dev processes found." exit 0 fi -echo "Found ${#pids[@]} Paperclip dev process(es):" -echo "" +if [[ ${#node_pids[@]} -gt 0 ]]; then + echo "Found ${#node_pids[@]} Paperclip dev node process(es):" + echo "" -for i in "${!pids[@]}"; do - line="${lines[$i]}" - pid=$(echo "$line" | awk '{print $2}') - start=$(echo "$line" | awk '{print $9}') - cmd=$(echo "$line" | awk '{for(i=11;i<=NF;i++) printf "%s ", $i; print ""}') - # Shorten the command for readability - cmd=$(echo "$cmd" | sed "s|$HOME/||g") - printf " PID %-7s started %-10s %s\n" "$pid" "$start" "$cmd" -done + for i in "${!node_pids[@]:-}"; do + line="${node_lines[$i]}" + pid=$(echo "$line" | awk '{print $2}') + start=$(echo "$line" | awk '{print $9}') + cmd=$(echo "$line" | awk '{for(i=11;i<=NF;i++) printf "%s ", $i; print ""}') + cmd=$(echo "$cmd" | sed "s|$HOME/||g") + printf " PID %-7s started %-10s %s\n" "$pid" "$start" "$cmd" + done -echo "" + echo "" +fi + +if [[ ${#pg_pids[@]} -gt 0 ]]; then + echo "Found ${#pg_pids[@]} embedded PostgreSQL master process(es):" + echo "" + + for i in "${!pg_pids[@]:-}"; do + pid="${pg_pids[$i]}" + data_dir="${pg_data_dirs[$i]}" + pidfile="${pg_pidfiles[$i]}" + short_data_dir="${data_dir/#$HOME\//}" + short_pidfile="${pidfile/#$HOME\//}" + printf " PID %-7s data %-55s pidfile %s\n" "$pid" "$short_data_dir" "$short_pidfile" + done + + echo "" +fi if [[ "$DRY_RUN" == true ]]; then echo "Dry run — re-run without --dry to kill these processes." exit 0 fi -echo "Sending SIGTERM..." -for pid in "${pids[@]}"; do - kill "$pid" 2>/dev/null && echo " killed $pid" || echo " $pid already gone" -done +if [[ ${#node_pids[@]} -gt 0 ]]; then + echo "Sending SIGTERM to Paperclip node processes..." + for pid in "${node_pids[@]}"; do + kill -TERM "$pid" 2>/dev/null && echo " signaled $pid" || echo " $pid already gone" + done + echo "Waiting briefly for node processes to exit..." + sleep 2 +fi -# Give processes a moment to exit, then SIGKILL any stragglers -sleep 2 -for pid in "${pids[@]}"; do - if kill -0 "$pid" 2>/dev/null; then - echo " $pid still alive, sending SIGKILL..." - kill -9 "$pid" 2>/dev/null || true +leftover_pg_pids=() +leftover_pg_data_dirs=() +for i in "${!pg_pids[@]:-}"; do + pid="${pg_pids[$i]}" + if is_pid_running "$pid"; then + leftover_pg_pids+=("$pid") + leftover_pg_data_dirs+=("${pg_data_dirs[$i]}") fi done +if [[ ${#leftover_pg_pids[@]} -gt 0 ]]; then + echo "Sending SIGTERM to leftover embedded PostgreSQL processes..." + for i in "${!leftover_pg_pids[@]:-}"; do + pid="${leftover_pg_pids[$i]}" + data_dir="${leftover_pg_data_dirs[$i]}" + kill -TERM "$pid" 2>/dev/null \ + && echo " signaled $pid ($data_dir)" \ + || echo " $pid already gone" + done + echo "Waiting up to 15s for PostgreSQL to shut down cleanly..." + for pid in "${leftover_pg_pids[@]:-}"; do + if wait_for_pid_exit "$pid" 15; then + echo " postgres $pid exited cleanly" + fi + done +fi + +if [[ ${#node_pids[@]} -gt 0 ]]; then + for pid in "${node_pids[@]:-}"; do + if kill -0 "$pid" 2>/dev/null; then + echo " node $pid still alive, sending SIGKILL..." + kill -KILL "$pid" 2>/dev/null || true + fi + done +fi + +if [[ ${#pg_pids[@]} -gt 0 ]]; then + for pid in "${pg_pids[@]:-}"; do + if kill -0 "$pid" 2>/dev/null; then + echo " postgres $pid still alive, sending SIGKILL..." + kill -KILL "$pid" 2>/dev/null || true + fi + done +fi + echo "Done." diff --git a/server/src/__tests__/heartbeat-workspace-session.test.ts b/server/src/__tests__/heartbeat-workspace-session.test.ts index 7fab2b429d..ca23d90795 100644 --- a/server/src/__tests__/heartbeat-workspace-session.test.ts +++ b/server/src/__tests__/heartbeat-workspace-session.test.ts @@ -3,11 +3,13 @@ import type { agents } from "@paperclipai/db"; import { sessionCodec as codexSessionCodec } from "@paperclipai/adapter-codex-local/server"; import { resolveDefaultAgentWorkspaceDir } from "../home-paths.js"; import { + applyPersistedExecutionWorkspaceConfig, buildExplicitResumeSessionOverride, formatRuntimeWorkspaceWarningLog, prioritizeProjectWorkspaceCandidatesForRun, parseSessionCompactionPolicy, resolveRuntimeSessionParamsForWorkspace, + stripWorkspaceRuntimeFromExecutionRunConfig, shouldResetTaskSessionForWake, type ResolvedWorkspaceForRun, } from "../services/heartbeat.ts"; @@ -120,6 +122,64 @@ describe("resolveRuntimeSessionParamsForWorkspace", () => { }); }); +describe("applyPersistedExecutionWorkspaceConfig", () => { + it("does not add workspace runtime when only the project workspace had manual runtime config", () => { + const result = applyPersistedExecutionWorkspaceConfig({ + config: {}, + workspaceConfig: null, + mode: "isolated_workspace", + }); + + expect("workspaceRuntime" in result).toBe(false); + }); + + it("applies explicit persisted execution workspace runtime config when present", () => { + const result = applyPersistedExecutionWorkspaceConfig({ + config: {}, + workspaceConfig: { + provisionCommand: null, + teardownCommand: null, + cleanupCommand: null, + desiredState: null, + workspaceRuntime: { + services: [{ name: "workspace-web" }], + }, + }, + mode: "isolated_workspace", + }); + + expect(result.workspaceRuntime).toEqual({ + services: [{ name: "workspace-web" }], + }); + }); +}); + +describe("stripWorkspaceRuntimeFromExecutionRunConfig", () => { + it("removes workspace runtime before heartbeat execution", () => { + const input = { + cwd: "/tmp/project", + workspaceStrategy: { + type: "git_worktree", + }, + workspaceRuntime: { + services: [{ name: "web" }], + }, + }; + + const result = stripWorkspaceRuntimeFromExecutionRunConfig(input); + + expect(result).toEqual({ + cwd: "/tmp/project", + workspaceStrategy: { + type: "git_worktree", + }, + }); + expect(input.workspaceRuntime).toEqual({ + services: [{ name: "web" }], + }); + }); +}); + describe("shouldResetTaskSessionForWake", () => { it("resets session context on assignment wake", () => { expect(shouldResetTaskSessionForWake({ wakeReason: "issue_assigned" })).toBe(true); diff --git a/server/src/index.ts b/server/src/index.ts index cfcde31a72..7ebfa7d1c5 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -28,7 +28,7 @@ import { createApp } from "./app.js"; import { loadConfig } from "./config.js"; import { logger } from "./middleware/logger.js"; import { setupLiveEventsWebSocketServer } from "./realtime/live-events-ws.js"; -import { heartbeatService, reconcilePersistedRuntimeServicesOnStartup, restartDesiredRuntimeServicesOnStartup, routineService } from "./services/index.js"; +import { heartbeatService, reconcilePersistedRuntimeServicesOnStartup, routineService } from "./services/index.js"; import { createStorageServiceFromConfig } from "./storage/index.js"; import { printStartupBanner } from "./startup-banner.js"; import { getBoardClaimWarningUrl, initializeBoardClaimChallenge } from "./board-claim.js"; @@ -557,15 +557,6 @@ export async function startServer(): Promise { "reconciled persisted runtime services from a previous server process", ); } - return restartDesiredRuntimeServicesOnStartup(db as any); - }) - .then((result) => { - if (result && result.restarted > 0) { - logger.warn( - { restarted: result.restarted, failed: result.failed }, - "restarted desired workspace runtime services on startup", - ); - } }) .catch((err) => { logger.error({ err }, "startup reconciliation of persisted runtime services failed"); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index e4c2b35be1..bc66f399fb 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -51,7 +51,6 @@ import { resolveExecutionWorkspaceMode, } from "./execution-workspace-policy.js"; import { instanceSettingsService } from "./instance-settings.js"; -import { readProjectWorkspaceRuntimeConfig } from "./project-workspace-runtime-config.js"; import { redactCurrentUserText, redactCurrentUserValue } from "../log-redaction.js"; import { hasSessionCompactionThresholds, @@ -77,10 +76,9 @@ const SESSIONED_LOCAL_ADAPTERS = new Set([ "pi_local", ]); -function applyPersistedExecutionWorkspaceConfig(input: { +export function applyPersistedExecutionWorkspaceConfig(input: { config: Record; workspaceConfig: ExecutionWorkspaceConfig | null; - projectWorkspaceRuntime: Record | null; mode: ReturnType; }) { const nextConfig = { ...input.config }; @@ -90,8 +88,6 @@ function applyPersistedExecutionWorkspaceConfig(input: { delete nextConfig.workspaceRuntime; } else if (input.workspaceConfig?.workspaceRuntime) { nextConfig.workspaceRuntime = { ...input.workspaceConfig.workspaceRuntime }; - } else if (input.projectWorkspaceRuntime) { - nextConfig.workspaceRuntime = { ...input.projectWorkspaceRuntime }; } } @@ -107,6 +103,12 @@ function applyPersistedExecutionWorkspaceConfig(input: { return nextConfig; } +export function stripWorkspaceRuntimeFromExecutionRunConfig(config: Record) { + const nextConfig = { ...config }; + delete nextConfig.workspaceRuntime; + return nextConfig; +} + function buildExecutionWorkspaceConfigSnapshot(config: Record): Partial | null { const strategy = parseObject(config.workspaceStrategy); const snapshot: Partial = {}; @@ -2114,35 +2116,19 @@ export function heartbeatService(db: Db) { : null; const existingExecutionWorkspace = issueRef?.executionWorkspaceId ? await executionWorkspacesSvc.getById(issueRef.executionWorkspaceId) : null; - const resolvedProjectWorkspace = - resolvedWorkspace.workspaceId - ? await db - .select({ metadata: projectWorkspaces.metadata }) - .from(projectWorkspaces) - .where( - and( - eq(projectWorkspaces.id, resolvedWorkspace.workspaceId), - eq(projectWorkspaces.companyId, agent.companyId), - ), - ) - .then((rows) => rows[0] ?? null) - : null; - const projectWorkspaceRuntimeConfig = readProjectWorkspaceRuntimeConfig( - (resolvedProjectWorkspace?.metadata as Record | null) ?? null, - ); const persistedWorkspaceManagedConfig = applyPersistedExecutionWorkspaceConfig({ config: workspaceManagedConfig, workspaceConfig: existingExecutionWorkspace?.config ?? null, - projectWorkspaceRuntime: projectWorkspaceRuntimeConfig?.workspaceRuntime ?? null, mode: executionWorkspaceMode, }); const mergedConfig = issueAssigneeOverrides?.adapterConfig ? { ...persistedWorkspaceManagedConfig, ...issueAssigneeOverrides.adapterConfig } : persistedWorkspaceManagedConfig; const configSnapshot = buildExecutionWorkspaceConfigSnapshot(mergedConfig); + const executionRunConfig = stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig); const { config: resolvedConfig, secretKeys } = await secretsSvc.resolveAdapterConfigForRuntime( agent.companyId, - mergedConfig, + executionRunConfig, ); const runtimeSkillEntries = await companySkills.listRuntimeSkillEntries(agent.companyId); const runtimeConfig = {