feat: add den worker runtime upgrades (#846)

This commit is contained in:
Omar McAdam
2026-03-11 18:30:50 -07:00
committed by GitHub
parent d70f0348d2
commit 73d716f6e8
7 changed files with 936 additions and 33 deletions

View File

@@ -50,6 +50,38 @@ export type OpenworkServerDiagnostics = {
tokenSource: { client: string; host: string };
};
export type OpenworkRuntimeServiceName = "openwork-server" | "opencode" | "opencode-router";
export type OpenworkRuntimeServiceSnapshot = {
name: OpenworkRuntimeServiceName;
enabled: boolean;
running: boolean;
targetVersion: string | null;
actualVersion: string | null;
upgradeAvailable: boolean;
};
export type OpenworkRuntimeSnapshot = {
ok: boolean;
orchestrator?: {
version: string;
startedAt: number;
};
worker?: {
workspace: string;
sandboxMode: string;
};
upgrade?: {
status: "idle" | "running" | "failed";
startedAt: number | null;
finishedAt: number | null;
error: string | null;
operationId: string | null;
services: OpenworkRuntimeServiceName[];
};
services: OpenworkRuntimeServiceSnapshot[];
};
export type OpenworkServerSettings = {
urlOverride?: string;
portOverride?: number;
@@ -1173,6 +1205,8 @@ export function createOpenworkServerClient(options: { baseUrl: string; token?: s
token,
health: () =>
requestJson<{ ok: boolean; version: string; uptimeMs: number }>(baseUrl, "/health", { token, hostToken, timeoutMs: timeouts.health }),
runtimeVersions: () =>
requestJson<OpenworkRuntimeSnapshot>(baseUrl, "/runtime/versions", { token, hostToken, timeoutMs: timeouts.status }),
status: () => requestJson<OpenworkServerDiagnostics>(baseUrl, "/status", { token, hostToken, timeoutMs: timeouts.status }),
capabilities: () => requestJson<OpenworkServerCapabilities>(baseUrl, "/capabilities", { token, hostToken, timeoutMs: timeouts.capabilities }),
opencodeRouterHealth: () =>

View File

@@ -20,12 +20,13 @@
{
"group": "Foundations",
"pages": [
"introduction",
"openwork",
"orbita-layout-style",
"opencode-router",
"cli"
]
"introduction",
"openwork",
"orbita-layout-style",
"opencode-router",
"cli",
"worker-runtime-upgrades"
]
},
{
"group": "Agent Playbooks",

View File

@@ -0,0 +1,55 @@
---
title: "Runtime upgrades"
description: "Inspect versions and refresh worker services"
---
Use this flow to check a Den worker's installed runtime versions and trigger an in-place service refresh.
---
## Inspect
Den exposes worker runtime inspection at:
- `GET /v1/workers/:id/runtime`
That endpoint proxies to the worker control endpoint:
- `GET /runtime/versions`
The response reports the worker runtime snapshot, including orchestrator metadata, service versions, and current upgrade state.
---
## Upgrade
Den exposes worker runtime upgrades at:
- `POST /v1/workers/:id/runtime/upgrade`
That endpoint proxies to the worker control endpoint:
- `POST /runtime/upgrade`
Default upgrades target:
- `openwork-server`
- `opencode`
You can also include `opencode-router` when that sidecar is enabled for the worker.
```json
{
"services": ["openwork-server", "opencode", "opencode-router"]
}
```
---
## Know limits
This flow upgrades worker runtime services only. It does not upgrade `openwork-orchestrator` itself.
It also does not perform migration steps, worker-to-worker data moves, or a Render rebuild/redeploy.
A separate follow-up PR will document and add the fuller migration and worker handoff flow.

View File

@@ -167,6 +167,28 @@ type BinaryDiagnostics = {
actualVersion?: string;
};
type RuntimeServiceName = "openwork-server" | "opencode" | "opencode-router";
type RuntimeServiceSnapshot = {
name: RuntimeServiceName;
enabled: boolean;
running: boolean;
source?: BinarySource;
path?: string;
targetVersion?: string;
actualVersion?: string;
upgradeAvailable: boolean;
};
type RuntimeUpgradeState = {
status: "idle" | "running" | "failed";
startedAt: number | null;
finishedAt: number | null;
error: string | null;
operationId: string | null;
services: RuntimeServiceName[];
};
type SidecarDiagnostics = {
dir: string;
baseUrl: string;
@@ -2775,6 +2797,8 @@ async function startOpenworkServer(options: {
opencodePassword?: string;
opencodeRouterHealthPort?: number;
opencodeRouterDataDir?: string;
controlBaseUrl?: string;
controlToken?: string;
logger: Logger;
runId: string;
logFormat: LogFormat;
@@ -2843,6 +2867,8 @@ async function startOpenworkServer(options: {
...(options.opencodeDirectory ? { OPENWORK_OPENCODE_DIRECTORY: options.opencodeDirectory } : {}),
...(options.opencodeUsername ? { OPENWORK_OPENCODE_USERNAME: options.opencodeUsername } : {}),
...(options.opencodePassword ? { OPENWORK_OPENCODE_PASSWORD: options.opencodePassword } : {}),
...(options.controlBaseUrl ? { OPENWORK_CONTROL_BASE_URL: options.controlBaseUrl } : {}),
...(options.controlToken ? { OPENWORK_CONTROL_TOKEN: options.controlToken } : {}),
},
});
@@ -3518,6 +3544,32 @@ async function verifyOpenworkServer(input: {
return actualVersion;
}
async function installGlobalPackages(packages: string[]): Promise<void> {
if (!packages.length) return;
await captureCommandOutput("npm", ["install", "-g", ...packages], { timeoutMs: 5 * 60_000 });
}
function buildRuntimeServiceSnapshot(input: {
name: RuntimeServiceName;
enabled: boolean;
running: boolean;
binary?: ResolvedBinary | null;
actualVersion?: string;
}): RuntimeServiceSnapshot {
const targetVersion = input.binary?.expectedVersion;
const actualVersion = input.actualVersion;
return {
name: input.name,
enabled: input.enabled,
running: input.enabled ? input.running : false,
source: input.binary?.source,
path: input.binary?.bin,
targetVersion,
actualVersion,
upgradeAvailable: Boolean(input.enabled && targetVersion && actualVersion && targetVersion !== actualVersion),
};
}
async function runChecks(input: {
opencodeClient: ReturnType<typeof createOpencodeClient>;
openworkUrl: string;
@@ -4295,7 +4347,7 @@ async function runRouterDaemon(args: ParsedArgs) {
`opencode hot reload: ${opencodeHotReload.enabled ? "on" : "off"} (debounce=${opencodeHotReload.debounceMs}ms cooldown=${opencodeHotReload.cooldownMs}ms)`,
);
logVerbose(`allow external: ${allowExternal ? "true" : "false"}`);
const opencodeBinary = await resolveOpencodeBin({
let opencodeBinary = await resolveOpencodeBin({
explicit: opencodeBin,
manifest,
allowExternal,
@@ -5194,7 +5246,7 @@ async function runStart(args: ParsedArgs) {
`opencode hot reload: ${opencodeHotReload.enabled ? "on" : "off"} (debounce=${opencodeHotReload.debounceMs}ms cooldown=${opencodeHotReload.cooldownMs}ms)`,
);
logVerbose(`allow external: ${allowExternal ? "true" : "false"}`);
const opencodeBinary = await resolveOpencodeBin({
let opencodeBinary = await resolveOpencodeBin({
explicit: explicitOpencodeBin,
manifest,
allowExternal,
@@ -5229,14 +5281,14 @@ async function runStart(args: ParsedArgs) {
false,
"OPENWORK_OPENCODE_ROUTER_REQUIRED",
);
const openworkServerBinary = await resolveOpenworkServerBin({
let openworkServerBinary = await resolveOpenworkServerBin({
explicit: explicitOpenworkServerBin,
manifest,
allowExternal,
sidecar,
source: sidecarSource,
});
const opencodeRouterBinary = opencodeRouterEnabled
let opencodeRouterBinary = opencodeRouterEnabled
? await resolveOpenCodeRouterBin({
explicit: explicitOpenCodeRouterBin,
manifest,
@@ -5299,8 +5351,251 @@ async function runStart(args: ParsedArgs) {
let sandboxStop: ((name: string) => Promise<void>) | null = null;
let sandboxStopCommand: string | null = null;
let sandboxCleanup: (() => Promise<void>) | null = null;
let opencodeChild: ChildProcess | null = null;
let openworkChild: ChildProcess | null = null;
let opencodeRouterChild: ChildProcess | null = null;
let controlServer: ReturnType<typeof createHttpServer> | null = null;
const controlPort = await resolvePort(undefined, "127.0.0.1");
const controlToken = randomUUID();
const controlBaseUrl = `http://127.0.0.1:${controlPort}`;
let opencodeActualVersion: string | undefined;
let openworkActualVersion: string | undefined;
const startedAt = Date.now();
let opencodeRouterHealthInterval: NodeJS.Timeout | null = null;
const restartingServices = new Set<string>();
const runtimeUpgradeState: RuntimeUpgradeState = {
status: "idle",
startedAt: null,
finishedAt: null,
error: null,
operationId: null,
services: [],
};
const removeChildHandle = (name: string) => {
const index = children.findIndex((handle) => handle.name === name);
if (index >= 0) children.splice(index, 1);
};
const getRuntimeSnapshot = () => {
const services = [
buildRuntimeServiceSnapshot({
name: "openwork-server",
enabled: true,
running: Boolean(openworkChild && isProcessAlive(openworkChild.pid)),
binary: openworkServerBinary,
actualVersion: openworkActualVersion,
}),
buildRuntimeServiceSnapshot({
name: "opencode",
enabled: true,
running: Boolean(opencodeChild && isProcessAlive(opencodeChild.pid)),
binary: opencodeBinary,
actualVersion: opencodeActualVersion,
}),
buildRuntimeServiceSnapshot({
name: "opencode-router",
enabled: Boolean(opencodeRouterEnabled && opencodeRouterBinary),
running: Boolean(opencodeRouterChild && isProcessAlive(opencodeRouterChild.pid)),
binary: opencodeRouterBinary,
actualVersion: opencodeRouterActualVersion,
}),
];
return {
ok: true,
orchestrator: {
version: cliVersion,
startedAt,
},
worker: {
workspace: resolvedWorkspace,
sandboxMode,
},
upgrade: {
...runtimeUpgradeState,
},
services,
};
};
const restartOpencode = async () => {
if (sandboxMode !== "none") {
throw new Error("Runtime upgrade is not supported while sandbox mode is enabled");
}
if (opencodeChild) {
restartingServices.add("opencode");
removeChildHandle("opencode");
await stopChild(opencodeChild);
opencodeChild = null;
}
opencodeActualVersion = await verifyOpencodeVersion(opencodeBinary);
const child = await startOpencode({
bin: opencodeBinary.bin,
workspace: resolvedWorkspace,
stateLayout: opencodeStateLayout,
hotReload: opencodeHotReload,
bindHost: opencodeBindHost,
port: opencodePort,
username: opencodeUsername,
password: opencodePassword,
corsOrigins: corsOrigins.length ? corsOrigins : ["*"],
logger,
runId,
logFormat,
opencodeRouterHealthPort: opencodeRouterEnabled ? opencodeRouterHealthPort : undefined,
});
opencodeChild = child;
children.push({ name: "opencode", child });
logger.info("Process spawned", { pid: child.pid ?? 0, cause: "runtime-upgrade" }, "opencode");
child.on("exit", (code, signal) => handleExit("opencode", code, signal));
child.on("error", (error) => handleSpawnError("opencode", error));
await waitForOpencodeHealthy(
createOpencodeClient({
baseUrl: opencodeBaseUrl,
directory: resolvedWorkspace,
headers: opencodeUsername && opencodePassword ? { Authorization: `Basic ${encodeBasicAuth(opencodeUsername, opencodePassword)}` } : undefined,
}),
);
};
const restartOpenworkServer = async () => {
if (sandboxMode !== "none") {
throw new Error("Runtime upgrade is not supported while sandbox mode is enabled");
}
if (openworkChild) {
restartingServices.add("openwork-server");
removeChildHandle("openwork-server");
await stopChild(openworkChild);
openworkChild = null;
}
const child = await startOpenworkServer({
bin: openworkServerBinary.bin,
host: openworkHost,
port: openworkPort,
workspace: resolvedWorkspace,
token: openworkToken,
hostToken: openworkHostToken,
approvalMode: approvalMode === "auto" ? "auto" : "manual",
approvalTimeoutMs,
readOnly,
corsOrigins: corsOrigins.length ? corsOrigins : ["*"],
opencodeBaseUrl: opencodeConnectUrl,
opencodeDirectory: resolvedWorkspace,
opencodeUsername,
opencodePassword,
opencodeRouterHealthPort: opencodeRouterChild ? opencodeRouterHealthPort : undefined,
opencodeRouterDataDir: opencodeRouterChild ? (opencodeRouterDataDir ?? undefined) : undefined,
logger,
runId,
logFormat,
controlBaseUrl,
controlToken,
});
openworkChild = child;
children.push({ name: "openwork-server", child });
logger.info("Process spawned", { pid: child.pid ?? 0, cause: "runtime-upgrade" }, "openwork-server");
child.on("exit", (code, signal) => handleExit("openwork-server", code, signal));
child.on("error", (error) => handleSpawnError("openwork-server", error));
await waitForHealthy(openworkBaseUrl);
openworkActualVersion = await verifyOpenworkServer({
baseUrl: openworkBaseUrl,
token: openworkToken,
hostToken: openworkHostToken,
expectedVersion: openworkServerBinary.expectedVersion,
expectedWorkspace: resolvedWorkspace,
expectedOpencodeBaseUrl: opencodeConnectUrl,
expectedOpencodeDirectory: resolvedWorkspace,
expectedOpencodeUsername: opencodeUsername,
expectedOpencodePassword: opencodePassword,
});
};
const restartOpenCodeRouter = async () => {
if (!opencodeRouterEnabled || !opencodeRouterBinary || sandboxMode !== "none") {
return;
}
if (opencodeRouterChild) {
restartingServices.add("opencode-router");
removeChildHandle("opencode-router");
await stopChild(opencodeRouterChild);
opencodeRouterChild = null;
}
opencodeRouterActualVersion = await verifyOpenCodeRouterVersion(opencodeRouterBinary);
opencodeRouterChild = await startOpenCodeRouter({
bin: opencodeRouterBinary.bin,
workspace: resolvedWorkspace,
opencodeUrl: opencodeConnectUrl,
opencodeUsername,
opencodePassword,
opencodeRouterHealthPort,
opencodeRouterDataDir: opencodeRouterDataDir ?? undefined,
logger,
runId,
logFormat,
});
children.push({ name: "opencode-router", child: opencodeRouterChild });
opencodeRouterChild.on("exit", (code, signal) => handleExit("opencode-router", code, signal));
opencodeRouterChild.on("error", (error) => handleSpawnError("opencode-router", error));
await waitForOpenCodeRouterHealthy(`http://127.0.0.1:${opencodeRouterHealthPort}`, 10_000, 400);
};
const performRuntimeUpgrade = async (services: RuntimeServiceName[]) => {
const opId = randomUUID();
runtimeUpgradeState.status = "running";
runtimeUpgradeState.startedAt = Date.now();
runtimeUpgradeState.finishedAt = null;
runtimeUpgradeState.error = null;
runtimeUpgradeState.operationId = opId;
runtimeUpgradeState.services = services;
try {
if (sandboxMode !== "none") {
throw new Error("Runtime upgrade is only supported for non-sandbox workers");
}
if (services.includes("openwork-server") && openworkServerBinary.source === "external" && openworkServerBinary.expectedVersion) {
await installGlobalPackages([`openwork-server@${openworkServerBinary.expectedVersion}`]);
}
if (services.includes("opencode-router") && opencodeRouterBinary?.source === "external" && opencodeRouterBinary.expectedVersion) {
await installGlobalPackages([`opencode-router@${opencodeRouterBinary.expectedVersion}`]);
}
if (services.includes("openwork-server")) {
openworkServerBinary = await resolveOpenworkServerBin({
explicit: explicitOpenworkServerBin,
manifest,
allowExternal,
sidecar,
source: sidecarSource,
});
}
if (services.includes("opencode")) {
opencodeBinary = await resolveOpencodeBin({
explicit: explicitOpencodeBin,
manifest,
allowExternal,
sidecar,
source: opencodeSource,
});
}
if (services.includes("opencode-router") && opencodeRouterEnabled) {
opencodeRouterBinary = await resolveOpenCodeRouterBin({
explicit: explicitOpenCodeRouterBin,
manifest,
allowExternal,
sidecar,
source: sidecarSource,
});
}
if (services.includes("opencode")) {
await restartOpencode();
}
if (services.includes("opencode-router")) {
await restartOpenCodeRouter();
}
if (services.includes("openwork-server") || services.includes("opencode")) {
await restartOpenworkServer();
}
runtimeUpgradeState.status = "idle";
runtimeUpgradeState.finishedAt = Date.now();
} catch (error) {
runtimeUpgradeState.status = "failed";
runtimeUpgradeState.finishedAt = Date.now();
runtimeUpgradeState.error = error instanceof Error ? error.message : String(error);
logger.error("Runtime upgrade failed", { error: runtimeUpgradeState.error, services }, "openwork-orchestrator");
}
};
const shutdown = async () => {
if (shuttingDown) return;
shuttingDown = true;
@@ -5310,6 +5605,10 @@ async function runStart(args: ParsedArgs) {
clearInterval(opencodeRouterHealthInterval);
opencodeRouterHealthInterval = null;
}
if (controlServer) {
await new Promise<void>((resolve) => controlServer?.close(() => resolve()));
controlServer = null;
}
logger.info(
"Shutting down",
{ children: children.map((handle) => handle.name) },
@@ -5510,6 +5809,10 @@ async function runStart(args: ParsedArgs) {
const handleExit = (name: string, code: number | null, signal: NodeJS.Signals | null) => {
if (shuttingDown || detached) return;
if (restartingServices.has(name)) {
restartingServices.delete(name);
return;
}
const reason = code !== null ? `code ${code}` : signal ? `signal ${signal}` : "unknown";
const services =
name === "sandbox"
@@ -5530,11 +5833,60 @@ async function runStart(args: ParsedArgs) {
};
try {
const opencodeActualVersion =
sandboxMode !== "none" ? opencodeBinary.expectedVersion : await verifyOpencodeVersion(opencodeBinary);
let openworkActualVersion: string | undefined;
opencodeActualVersion = sandboxMode !== "none" ? opencodeBinary.expectedVersion : await verifyOpencodeVersion(opencodeBinary);
let opencodeClient: ReturnType<typeof createOpencodeClient>;
controlServer = createHttpServer(async (req, res) => {
const method = req.method ?? "GET";
const url = new URL(req.url ?? "/", controlBaseUrl);
res.setHeader("Content-Type", "application/json");
const authHeader = req.headers.authorization ?? "";
if (authHeader !== `Bearer ${controlToken}`) {
res.statusCode = 401;
res.end(JSON.stringify({ ok: false, error: "unauthorized" }));
return;
}
if (method === "GET" && url.pathname === "/runtime/versions") {
res.statusCode = 200;
res.end(JSON.stringify(getRuntimeSnapshot()));
return;
}
if (method === "POST" && url.pathname === "/runtime/upgrade") {
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
let body: { services?: RuntimeServiceName[] } | null = null;
try {
body = chunks.length ? (JSON.parse(Buffer.concat(chunks).toString("utf8")) as { services?: RuntimeServiceName[] }) : null;
} catch {
body = null;
}
const requested = Array.isArray(body?.services) ? body.services : ["openwork-server", "opencode"];
const services = Array.from(new Set(requested.filter((item): item is RuntimeServiceName => item === "openwork-server" || item === "opencode" || item === "opencode-router")));
if (!services.length) {
res.statusCode = 400;
res.end(JSON.stringify({ ok: false, error: "invalid_services" }));
return;
}
if (runtimeUpgradeState.status === "running") {
res.statusCode = 409;
res.end(JSON.stringify({ ok: false, error: "upgrade_in_progress", upgrade: runtimeUpgradeState }));
return;
}
res.statusCode = 202;
res.end(JSON.stringify({ ok: true, started: true, services, upgrade: { ...runtimeUpgradeState, status: "running" } }));
void performRuntimeUpgrade(services);
return;
}
res.statusCode = 404;
res.end(JSON.stringify({ ok: false, error: "not_found" }));
});
await new Promise<void>((resolve, reject) => {
controlServer?.once("error", reject);
controlServer?.listen(controlPort, "127.0.0.1", () => resolve());
});
if (sandboxMode !== "none") {
const containerName = `openwork-orchestrator-${runId.replace(/[^a-zA-Z0-9_.-]+/g, "-").slice(0, 24)}`;
sandboxContainerName = containerName;
@@ -5688,7 +6040,7 @@ async function runStart(args: ParsedArgs) {
}
logVerbose(`openwork-server version: ${openworkActualVersion ?? "unknown"}`);
} else {
const opencodeChild = await startOpencode({
const startedOpencodeChild = await startOpencode({
bin: opencodeBinary.bin,
workspace: resolvedWorkspace,
stateLayout: opencodeStateLayout,
@@ -5703,15 +6055,16 @@ async function runStart(args: ParsedArgs) {
logFormat,
opencodeRouterHealthPort: opencodeRouterEnabled ? opencodeRouterHealthPort : undefined,
});
children.push({ name: "opencode", child: opencodeChild });
opencodeChild = startedOpencodeChild;
children.push({ name: "opencode", child: startedOpencodeChild });
tui?.updateService("opencode", {
status: "running",
pid: opencodeChild.pid ?? undefined,
pid: startedOpencodeChild.pid ?? undefined,
port: opencodePort,
});
logger.info("Process spawned", { pid: opencodeChild.pid ?? 0 }, "opencode");
opencodeChild.on("exit", (code, signal) => handleExit("opencode", code, signal));
opencodeChild.on("error", (error) => handleSpawnError("opencode", error));
logger.info("Process spawned", { pid: startedOpencodeChild.pid ?? 0 }, "opencode");
startedOpencodeChild.on("exit", (code, signal) => handleExit("opencode", code, signal));
startedOpencodeChild.on("error", (error) => handleSpawnError("opencode", error));
const authHeaders: Record<string, string> = {};
if (opencodeUsername && opencodePassword) {
@@ -5728,7 +6081,6 @@ async function runStart(args: ParsedArgs) {
logger.info("Healthy", { url: opencodeBaseUrl }, "opencode");
tui?.updateService("opencode", { status: "healthy" });
let opencodeRouterChild: ChildProcess | null = null;
let opencodeRouterReady = false;
if (opencodeRouterEnabled) {
if (!opencodeRouterBinary) {
@@ -5738,7 +6090,7 @@ async function runStart(args: ParsedArgs) {
logVerbose(`opencodeRouter version: ${opencodeRouterActualVersion ?? "unknown"}`);
try {
opencodeRouterChild = await startOpenCodeRouter({
const startedOpenCodeRouterChild = await startOpenCodeRouter({
bin: opencodeRouterBinary.bin,
workspace: resolvedWorkspace,
opencodeUrl: opencodeConnectUrl,
@@ -5750,14 +6102,19 @@ async function runStart(args: ParsedArgs) {
runId,
logFormat,
});
children.push({ name: "opencode-router", child: opencodeRouterChild });
opencodeRouterChild = startedOpenCodeRouterChild;
children.push({ name: "opencode-router", child: startedOpenCodeRouterChild });
tui?.updateService("router", {
status: "running",
pid: opencodeRouterChild.pid ?? undefined,
pid: startedOpenCodeRouterChild.pid ?? undefined,
port: opencodeRouterHealthPort,
});
logger.info("Process spawned", { pid: opencodeRouterChild.pid ?? 0 }, "opencode-router");
opencodeRouterChild.on("exit", (code, signal) => {
logger.info("Process spawned", { pid: startedOpenCodeRouterChild.pid ?? 0 }, "opencode-router");
startedOpenCodeRouterChild.on("exit", (code, signal) => {
if (restartingServices.has("opencode-router")) {
restartingServices.delete("opencode-router");
return;
}
if (opencodeRouterRequired) {
handleExit("opencode-router", code, signal);
return;
@@ -5766,7 +6123,7 @@ async function runStart(args: ParsedArgs) {
tui?.updateService("router", { status: "stopped", message: reason });
logger.warn("Process exited, continuing without opencodeRouter", { reason, code, signal }, "opencode-router");
});
opencodeRouterChild.on("error", (error) => handleSpawnError("opencode-router", error));
startedOpenCodeRouterChild.on("error", (error) => handleSpawnError("opencode-router", error));
const healthBaseUrl = `http://127.0.0.1:${opencodeRouterHealthPort}`;
logger.info("Waiting for health", { url: healthBaseUrl }, "opencode-router");
@@ -5794,7 +6151,7 @@ async function runStart(args: ParsedArgs) {
}
}
const openworkChild = await startOpenworkServer({
const startedOpenworkChild = await startOpenworkServer({
bin: openworkServerBinary.bin,
host: openworkHost,
port: openworkPort,
@@ -5814,16 +6171,19 @@ async function runStart(args: ParsedArgs) {
logger,
runId,
logFormat,
controlBaseUrl,
controlToken,
});
children.push({ name: "openwork-server", child: openworkChild });
openworkChild = startedOpenworkChild;
children.push({ name: "openwork-server", child: startedOpenworkChild });
tui?.updateService("openwork-server", {
status: "running",
pid: openworkChild.pid ?? undefined,
pid: startedOpenworkChild.pid ?? undefined,
port: openworkPort,
});
logger.info("Process spawned", { pid: openworkChild.pid ?? 0 }, "openwork-server");
openworkChild.on("exit", (code, signal) => handleExit("openwork-server", code, signal));
openworkChild.on("error", (error) => handleSpawnError("openwork-server", error));
logger.info("Process spawned", { pid: startedOpenworkChild.pid ?? 0 }, "openwork-server");
startedOpenworkChild.on("exit", (code, signal) => handleExit("openwork-server", code, signal));
startedOpenworkChild.on("error", (error) => handleSpawnError("openwork-server", error));
logger.info("Waiting for health", { url: openworkBaseUrl }, "openwork-server");
await waitForHealthy(openworkBaseUrl);

View File

@@ -1395,6 +1395,28 @@ function createRoutes(config: ServerConfig, approvals: ApprovalService, tokens:
});
});
addRoute(routes, "GET", "/runtime/versions", "client", async () => {
const snapshot = await fetchRuntimeControl("/runtime/versions");
return jsonResponse(snapshot);
});
addRoute(routes, "POST", "/runtime/upgrade", "host", async (ctx) => {
const body = await readJsonBody(ctx.request);
const result = await fetchRuntimeControl("/runtime/upgrade", { method: "POST", body });
return jsonResponse(result, 202);
});
addRoute(routes, "GET", "/w/:id/runtime/versions", "client", async () => {
const snapshot = await fetchRuntimeControl("/runtime/versions");
return jsonResponse(snapshot);
});
addRoute(routes, "POST", "/w/:id/runtime/upgrade", "host", async (ctx) => {
const body = await readJsonBody(ctx.request);
const result = await fetchRuntimeControl("/runtime/upgrade", { method: "POST", body });
return jsonResponse(result, 202);
});
addRoute(routes, "GET", "/whoami", "client", async (ctx) => {
return jsonResponse({ ok: true, actor: ctx.actor ?? null });
});
@@ -4708,6 +4730,34 @@ async function updateOpenCodeRouterTelegramToken(
return response;
}
function getRuntimeControlConfig(): { baseUrl: string; token: string } | null {
const baseUrl = process.env.OPENWORK_CONTROL_BASE_URL?.trim() ?? "";
const token = process.env.OPENWORK_CONTROL_TOKEN?.trim() ?? "";
if (!baseUrl || !token) return null;
return { baseUrl: baseUrl.replace(/\/+$/, ""), token };
}
async function fetchRuntimeControl(path: string, init?: { method?: string; body?: unknown }) {
const control = getRuntimeControlConfig();
if (!control) {
throw new ApiError(501, "runtime_upgrade_unavailable", "Worker runtime control is not configured on this host");
}
const response = await fetch(`${control.baseUrl}${path}`, {
method: init?.method ?? "GET",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${control.token}`,
},
body: init?.body === undefined ? undefined : JSON.stringify(init.body),
});
const text = await response.text();
const json = text ? JSON.parse(text) : null;
if (!response.ok) {
throw new ApiError(response.status, "runtime_upgrade_failed", "Worker runtime control request failed", json);
}
return json;
}
async function updateOpenCodeRouterSlackTokens(
botToken: string,
appToken: string,

View File

@@ -96,6 +96,27 @@ type WorkerListItem = {
createdAt: string | null;
};
type RuntimeServiceName = "openwork-server" | "opencode" | "opencode-router";
type WorkerRuntimeService = {
name: RuntimeServiceName;
enabled: boolean;
running: boolean;
targetVersion: string | null;
actualVersion: string | null;
upgradeAvailable: boolean;
};
type WorkerRuntimeSnapshot = {
services: WorkerRuntimeService[];
upgrade: {
status: "idle" | "running" | "failed";
startedAt: string | null;
finishedAt: string | null;
error: string | null;
};
};
type EventLevel = "info" | "success" | "warning" | "error";
type LaunchEvent = {
@@ -452,6 +473,55 @@ function getWorkerTokens(payload: unknown): WorkerTokens | null {
return { clientToken, hostToken, openworkUrl, workspaceId };
}
function getWorkerRuntimeSnapshot(payload: unknown): WorkerRuntimeSnapshot | null {
if (!isRecord(payload) || !Array.isArray(payload.services)) {
return null;
}
const services = payload.services
.map((value) => {
if (!isRecord(value) || typeof value.name !== "string") {
return null;
}
return {
name: value.name as RuntimeServiceName,
enabled: value.enabled === true,
running: value.running === true,
targetVersion: typeof value.targetVersion === "string" ? value.targetVersion : null,
actualVersion: typeof value.actualVersion === "string" ? value.actualVersion : null,
upgradeAvailable: value.upgradeAvailable === true
};
})
.filter((item): item is WorkerRuntimeService => item !== null);
const upgrade = isRecord(payload.upgrade) ? payload.upgrade : null;
return {
services,
upgrade: {
status:
upgrade?.status === "running" || upgrade?.status === "failed" || upgrade?.status === "idle"
? upgrade.status
: "idle",
startedAt: typeof upgrade?.startedAt === "number" ? new Date(upgrade.startedAt).toISOString() : null,
finishedAt: typeof upgrade?.finishedAt === "number" ? new Date(upgrade.finishedAt).toISOString() : null,
error: typeof upgrade?.error === "string" ? upgrade.error : null
}
};
}
function getRuntimeServiceLabel(name: RuntimeServiceName): string {
switch (name) {
case "openwork-server":
return "OpenWork server";
case "opencode":
return "OpenCode";
case "opencode-router":
return "OpenCode Router";
}
}
function getBillingPrice(value: unknown): BillingPrice | null {
if (!isRecord(value)) {
return null;
@@ -994,6 +1064,10 @@ export function CloudControlPanel() {
const [showLaunchForm, setShowLaunchForm] = useState(false);
const [openAccordion, setOpenAccordion] = useState<"connect" | "actions" | "advanced" | null>(null);
const [showAdvancedOptions, setShowAdvancedOptions] = useState(false);
const [runtimeSnapshot, setRuntimeSnapshot] = useState<WorkerRuntimeSnapshot | null>(null);
const [runtimeBusy, setRuntimeBusy] = useState(false);
const [runtimeError, setRuntimeError] = useState<string | null>(null);
const [runtimeUpgradeBusy, setRuntimeUpgradeBusy] = useState(false);
const selectedWorker = workers.find((item) => item.workerId === workerLookupId) ?? null;
const activeWorker: WorkerLaunch | null =
@@ -1044,6 +1118,7 @@ export function CloudControlPanel() {
const effectiveCheckoutUrl = checkoutUrl ?? billingSummary?.checkoutUrl ?? null;
const billingSubscription = billingSummary?.subscription ?? null;
const billingPrice = billingSummary?.price ?? null;
const runtimeUpgradeCount = runtimeSnapshot?.services.filter((item) => item.upgradeAvailable).length ?? 0;
function appendEvent(level: EventLevel, label: string, detail: string) {
setEvents((current) => {
@@ -1161,6 +1236,105 @@ export function CloudControlPanel() {
}
}
async function refreshRuntime(workerId?: string, options: { quiet?: boolean } = {}) {
const targetWorkerId = workerId ?? activeWorker?.workerId ?? selectedWorker?.workerId ?? null;
if (!user || !targetWorkerId) {
setRuntimeSnapshot(null);
if (!options.quiet) {
setRuntimeError("Select a worker to inspect runtime versions.");
}
return null;
}
setRuntimeBusy(true);
if (!options.quiet) {
setRuntimeError(null);
}
try {
const { response, payload } = await requestJson(`/v1/workers/${encodeURIComponent(targetWorkerId)}/runtime`, {
method: "GET",
headers: authToken ? { Authorization: `Bearer ${authToken}` } : undefined
}, 12000);
if (!response.ok) {
const message = getErrorMessage(payload, `Runtime check failed with ${response.status}.`);
if (!options.quiet) {
setRuntimeError(message);
}
return null;
}
const snapshot = getWorkerRuntimeSnapshot(payload);
if (!snapshot) {
if (!options.quiet) {
setRuntimeError("Runtime details were missing from the worker response.");
}
return null;
}
setRuntimeSnapshot(snapshot);
return snapshot;
} catch (error) {
const message = error instanceof Error ? error.message : "Unknown network error";
if (!options.quiet) {
setRuntimeError(message);
}
return null;
} finally {
setRuntimeBusy(false);
}
}
async function handleRuntimeUpgrade() {
const targetWorkerId = activeWorker?.workerId ?? selectedWorker?.workerId ?? null;
if (!user || !targetWorkerId || runtimeUpgradeBusy) {
return;
}
setRuntimeUpgradeBusy(true);
setRuntimeError(null);
try {
const { response, payload } = await requestJson(`/v1/workers/${encodeURIComponent(targetWorkerId)}/runtime/upgrade`, {
method: "POST",
headers: authToken ? { Authorization: `Bearer ${authToken}` } : undefined,
body: JSON.stringify({ services: ["openwork-server", "opencode"] })
}, 12000);
if (!response.ok) {
const message = getErrorMessage(payload, `Runtime upgrade failed with ${response.status}.`);
setRuntimeError(message);
appendEvent("error", "Runtime upgrade failed", message);
return;
}
appendEvent("info", "Runtime upgrade started", activeWorker?.workerName ?? selectedWorker?.workerName ?? targetWorkerId);
setRuntimeSnapshot((current) => current
? {
...current,
upgrade: {
...current.upgrade,
status: "running",
startedAt: new Date().toISOString(),
finishedAt: null,
error: null
}
}
: current);
window.setTimeout(() => {
void refreshRuntime(targetWorkerId, { quiet: true });
}, 4000);
} catch (error) {
const message = error instanceof Error ? error.message : "Unknown network error";
setRuntimeError(message);
appendEvent("error", "Runtime upgrade failed", message);
} finally {
setRuntimeUpgradeBusy(false);
}
}
async function refreshBilling(options: { includeCheckout?: boolean; quiet?: boolean } = {}) {
if (!user) {
setBillingSummary(null);
@@ -1356,6 +1530,30 @@ export function CloudControlPanel() {
void refreshWorkers();
}, [user?.id, authToken]);
useEffect(() => {
const targetWorkerId = activeWorker?.workerId ?? selectedWorker?.workerId ?? null;
if (!user || !targetWorkerId) {
setRuntimeSnapshot(null);
setRuntimeError(null);
return;
}
void refreshRuntime(targetWorkerId, { quiet: true });
}, [user?.id, authToken, activeWorker?.workerId, selectedWorker?.workerId]);
useEffect(() => {
const targetWorkerId = activeWorker?.workerId ?? selectedWorker?.workerId ?? null;
if (!targetWorkerId || runtimeSnapshot?.upgrade.status !== "running") {
return;
}
const timer = window.setInterval(() => {
void refreshRuntime(targetWorkerId, { quiet: true });
}, 5000);
return () => window.clearInterval(timer);
}, [activeWorker?.workerId, selectedWorker?.workerId, runtimeSnapshot?.upgrade.status]);
useEffect(() => {
if (!user) {
setBillingSummary(null);
@@ -2495,7 +2693,7 @@ export function CloudControlPanel() {
</h2>
<p className="mb-6 text-sm text-slate-500">{getWorkerStatusCopy(selectedWorkerStatus)}</p>
<div className="grid grid-cols-1 gap-4 md:grid-cols-2">
<div className="grid grid-cols-1 gap-4 md:grid-cols-3">
<div className="rounded-[20px] border border-slate-100 bg-white p-4">
<p className="text-sm font-medium text-slate-500">Status</p>
<p className="mt-2 text-2xl font-bold text-slate-900">{selectedStatusMeta.label}</p>
@@ -2504,6 +2702,79 @@ export function CloudControlPanel() {
<p className="text-sm font-medium text-slate-500">Connection</p>
<p className="mt-2 text-2xl font-bold text-slate-900">{openworkDeepLink ? "Ready" : "Preparing"}</p>
</div>
<div className="rounded-[20px] border border-slate-100 bg-white p-4">
<p className="text-sm font-medium text-slate-500">Runtime</p>
<p className="mt-2 text-2xl font-bold text-slate-900">
{runtimeBusy ? "Checking" : runtimeUpgradeCount > 0 ? `${runtimeUpgradeCount} update${runtimeUpgradeCount === 1 ? "" : "s"}` : "Current"}
</p>
</div>
</div>
</div>
<div className="rounded-[28px] border border-slate-100 bg-white p-6">
<div className="mb-5 flex flex-col gap-4 md:flex-row md:items-center md:justify-between">
<div>
<h3 className="text-lg font-bold tracking-tight text-slate-900">Worker runtime</h3>
<p className="text-sm text-slate-500">Compare installed runtime versions with the versions this worker should be running.</p>
</div>
<div className="flex flex-wrap items-center gap-2">
<button
type="button"
className="rounded-[12px] border border-slate-200 bg-white px-4 py-2 text-sm font-semibold text-slate-700 transition hover:bg-slate-50 disabled:cursor-not-allowed disabled:opacity-50"
onClick={() => void refreshRuntime(selectedWorker.workerId)}
disabled={runtimeBusy || runtimeUpgradeBusy}
>
{runtimeBusy ? "Checking..." : "Refresh runtime"}
</button>
<button
type="button"
className="rounded-[12px] bg-[#1B29FF] px-4 py-2 text-sm font-semibold text-white transition hover:bg-[#151FDA] disabled:cursor-not-allowed disabled:opacity-50"
onClick={() => void handleRuntimeUpgrade()}
disabled={runtimeUpgradeBusy || runtimeBusy || selectedStatusMeta.bucket !== "ready"}
>
{runtimeUpgradeBusy || runtimeSnapshot?.upgrade.status === "running" ? "Upgrading..." : "Upgrade runtime"}
</button>
</div>
</div>
{runtimeError ? (
<div className="mb-4 rounded-[14px] border border-rose-200 bg-rose-50 px-4 py-3 text-sm text-rose-700">{runtimeError}</div>
) : null}
{runtimeSnapshot?.upgrade.status === "failed" && runtimeSnapshot.upgrade.error ? (
<div className="mb-4 rounded-[14px] border border-amber-200 bg-amber-50 px-4 py-3 text-sm text-amber-800">
Last upgrade failed: {runtimeSnapshot.upgrade.error}
</div>
) : null}
{runtimeUpgradeCount > 0 ? (
<div className="mb-4 rounded-[14px] border border-amber-200 bg-amber-50 px-4 py-3 text-sm text-amber-800">
This worker has {runtimeUpgradeCount} runtime component{runtimeUpgradeCount === 1 ? "" : "s"} behind the target version.
</div>
) : null}
<div className="space-y-3">
{(runtimeSnapshot?.services ?? []).map((service) => (
<div key={service.name} className="flex flex-col gap-3 rounded-[18px] border border-slate-100 bg-slate-50 px-4 py-3 md:flex-row md:items-center md:justify-between">
<div>
<p className="text-sm font-semibold text-slate-900">{getRuntimeServiceLabel(service.name)}</p>
<p className="text-xs text-slate-500">
Installed {service.actualVersion ?? "unknown"} · Target {service.targetVersion ?? "unknown"}
</p>
</div>
<div className="flex items-center gap-2 text-xs font-semibold uppercase tracking-wide">
<span className={`rounded-full px-2.5 py-1 ${service.running ? "bg-emerald-100 text-emerald-700" : "bg-slate-200 text-slate-600"}`}>
{service.running ? "Running" : service.enabled ? "Stopped" : "Disabled"}
</span>
<span className={`rounded-full px-2.5 py-1 ${service.upgradeAvailable ? "bg-amber-100 text-amber-700" : "bg-slate-200 text-slate-600"}`}>
{service.upgradeAvailable ? "Upgrade available" : "Current"}
</span>
</div>
</div>
))}
{!runtimeSnapshot && !runtimeBusy ? (
<p className="text-sm text-slate-500">Runtime details appear after the worker is reachable.</p>
) : null}
</div>
</div>

View File

@@ -142,6 +142,78 @@ async function resolveConnectUrlFromCandidates(workerId: string, instanceUrl: st
return null
}
async function getWorkerRuntimeAccess(workerId: string) {
const instance = await getLatestWorkerInstance(workerId)
const tokenRows = await db
.select()
.from(WorkerTokenTable)
.where(and(eq(WorkerTokenTable.worker_id, workerId), isNull(WorkerTokenTable.revoked_at)))
.orderBy(asc(WorkerTokenTable.created_at))
const hostToken = tokenRows.find((entry) => entry.scope === "host")?.token ?? null
if (!instance?.url || !hostToken) {
return null
}
return {
instance,
hostToken,
candidates: getConnectUrlCandidates(workerId, instance.url),
}
}
async function fetchWorkerRuntimeJson(input: {
workerId: string
path: string
method?: "GET" | "POST"
body?: unknown
}) {
const access = await getWorkerRuntimeAccess(input.workerId)
if (!access) {
return {
ok: false as const,
status: 409,
payload: {
error: "worker_runtime_unavailable",
message: "Worker runtime access is not ready yet. Wait for provisioning to finish and try again.",
},
}
}
let lastPayload: unknown = null
let lastStatus = 502
for (const candidate of access.candidates) {
try {
const response = await fetch(`${normalizeUrl(candidate)}${input.path}`, {
method: input.method ?? "GET",
headers: {
Accept: "application/json",
"Content-Type": "application/json",
"X-OpenWork-Host-Token": access.hostToken,
},
body: input.body === undefined ? undefined : JSON.stringify(input.body),
})
const text = await response.text()
lastStatus = response.status
try {
lastPayload = text ? JSON.parse(text) : null
} catch {
lastPayload = text ? { message: text } : null
}
if (response.ok) {
return { ok: true as const, status: response.status, payload: lastPayload }
}
} catch (error) {
lastPayload = { message: error instanceof Error ? error.message : "worker_request_failed" }
}
}
return { ok: false as const, status: lastStatus, payload: lastPayload }
}
async function requireSession(req: express.Request, res: express.Response) {
const session = await auth.api.getSession({
headers: fromNodeHeaders(req.headers),
@@ -549,6 +621,66 @@ workersRouter.post("/:id/tokens", asyncRoute(async (req, res) => {
})
}))
workersRouter.get("/:id/runtime", asyncRoute(async (req, res) => {
const session = await requireSession(req, res)
if (!session) return
const orgId = await getOrgId(session.user.id)
if (!orgId) {
res.status(404).json({ error: "worker_not_found" })
return
}
const rows = await db
.select()
.from(WorkerTable)
.where(and(eq(WorkerTable.id, req.params.id), eq(WorkerTable.org_id, orgId)))
.limit(1)
if (rows.length === 0) {
res.status(404).json({ error: "worker_not_found" })
return
}
const runtime = await fetchWorkerRuntimeJson({
workerId: rows[0].id,
path: "/runtime/versions",
})
res.status(runtime.status).json(runtime.payload)
}))
workersRouter.post("/:id/runtime/upgrade", asyncRoute(async (req, res) => {
const session = await requireSession(req, res)
if (!session) return
const orgId = await getOrgId(session.user.id)
if (!orgId) {
res.status(404).json({ error: "worker_not_found" })
return
}
const rows = await db
.select()
.from(WorkerTable)
.where(and(eq(WorkerTable.id, req.params.id), eq(WorkerTable.org_id, orgId)))
.limit(1)
if (rows.length === 0) {
res.status(404).json({ error: "worker_not_found" })
return
}
const runtime = await fetchWorkerRuntimeJson({
workerId: rows[0].id,
path: "/runtime/upgrade",
method: "POST",
body: req.body ?? {},
})
res.status(runtime.status).json(runtime.payload)
}))
workersRouter.delete("/:id", asyncRoute(async (req, res) => {
const session = await requireSession(req, res)
if (!session) return