Improve heartbeat execution, run tracking, and agent detail display

Enhance heartbeat service with better process adapter error recovery
and run state management. Expand heartbeat-run CLI with additional
output and diagnostics. Improve AgentDetail page run history and
status display.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

cli/src/commands/heartbeat-run.ts

@@ -26,6 +26,29 @@ interface HeartbeatRunOptions {
   debug?: boolean;
 }
 
+function asRecord(value: unknown): Record<string, unknown> | null {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null;
+}
+
+function asErrorText(value: unknown): string {
+  if (typeof value === "string") return value;
+  const obj = asRecord(value);
+  if (!obj) return "";
+  const message =
+    (typeof obj.message === "string" && obj.message) ||
+    (typeof obj.error === "string" && obj.error) ||
+    (typeof obj.code === "string" && obj.code) ||
+    "";
+  if (message) return message;
+  try {
+    return JSON.stringify(obj);
+  } catch {
+    return "";
+  }
+}
+
 export async function heartbeatRun(opts: HeartbeatRunOptions): Promise<void> {
   const debug = Boolean(opts.debug);
   const parsedTimeout = Number.parseInt(opts.timeoutMs, 10);
@@ -185,11 +208,20 @@ export async function heartbeatRun(opts: HeartbeatRunOptions): Promise<void> {
       const output = Number(usage.output_tokens ?? 0);
       const cached = Number(usage.cache_read_input_tokens ?? 0);
       const cost = Number(parsed.total_cost_usd ?? 0);
+      const subtype = typeof parsed.subtype === "string" ? parsed.subtype : "";
+      const isError = parsed.is_error === true;
       const resultText = typeof parsed.result === "string" ? parsed.result : "";
       if (resultText) {
         console.log(pc.green("result:"));
         console.log(resultText);
       }
+      const errors = Array.isArray(parsed.errors) ? parsed.errors.map(asErrorText).filter(Boolean) : [];
+      if (subtype.startsWith("error") || isError || errors.length > 0) {
+        console.log(pc.red(`claude_result: subtype=${subtype || "unknown"} is_error=${isError ? "true" : "false"}`));
+        if (errors.length > 0) {
+          console.log(pc.red(`claude_errors: ${errors.join(" | ")}`));
+        }
+      }
       console.log(
         pc.blue(
           `tokens: in=${Number.isFinite(input) ? input : 0} out=${Number.isFinite(output) ? output : 0} cached=${Number.isFinite(cached) ? cached : 0} cost=$${Number.isFinite(cost) ? cost.toFixed(6) : "0.000000"}`,
@@ -255,6 +287,7 @@ export async function heartbeatRun(opts: HeartbeatRunOptions): Promise<void> {
   activeRunId = runId;
   let finalStatus: string | null = null;
   let finalError: string | null = null;
+  let finalRun: HeartbeatRun | null = null;
 
   const deadline = timeoutMs > 0 ? Date.now() + timeoutMs : null;
   if (!activeRunId) {
@@ -290,6 +323,7 @@ export async function heartbeatRun(opts: HeartbeatRunOptions): Promise<void> {
     if (currentStatus && TERMINAL_STATUSES.has(currentStatus)) {
       finalStatus = currentRun.status;
       finalError = currentRun.error;
+      finalRun = currentRun;
       break;
     }
 
@@ -337,6 +371,33 @@ export async function heartbeatRun(opts: HeartbeatRunOptions): Promise<void> {
     if (finalError) {
       console.log(pc.red(`Error: ${finalError}`));
     }
+    if (finalRun) {
+      const resultObj = asRecord(finalRun.resultJson);
+      if (resultObj) {
+        const subtype = typeof resultObj.subtype === "string" ? resultObj.subtype : "";
+        const isError = resultObj.is_error === true;
+        const errors = Array.isArray(resultObj.errors) ? resultObj.errors.map(asErrorText).filter(Boolean) : [];
+        const resultText = typeof resultObj.result === "string" ? resultObj.result.trim() : "";
+        if (subtype || isError || errors.length > 0 || resultText) {
+          console.log(pc.red("Claude result details:"));
+          if (subtype) console.log(pc.red(`  subtype: ${subtype}`));
+          if (isError) console.log(pc.red("  is_error: true"));
+          if (errors.length > 0) console.log(pc.red(`  errors: ${errors.join(" | ")}`));
+          if (resultText) console.log(pc.red(`  result: ${resultText}`));
+        }
+      }
+
+      const stderrExcerpt = typeof finalRun.stderrExcerpt === "string" ? finalRun.stderrExcerpt.trim() : "";
+      const stdoutExcerpt = typeof finalRun.stdoutExcerpt === "string" ? finalRun.stdoutExcerpt.trim() : "";
+      if (stderrExcerpt) {
+        console.log(pc.red("stderr excerpt:"));
+        console.log(stderrExcerpt);
+      }
+      if (stdoutExcerpt && (debug || !stderrExcerpt)) {
+        console.log(pc.gray("stdout excerpt:"));
+        console.log(stdoutExcerpt);
+      }
+    }
     process.exitCode = 1;
   } else {
     process.exitCode = 1;

server/src/services/heartbeat.ts

@@ -47,6 +47,7 @@ interface AdapterExecutionResult {
   costUsd?: number | null;
   resultJson?: Record<string, unknown> | null;
   summary?: string | null;
+  clearSession?: boolean;
 }
 
 interface AdapterInvocationMeta {
@@ -188,6 +189,65 @@ function parseCodexJsonl(stdout: string) {
   };
 }
 
+function describeClaudeFailure(parsed: Record<string, unknown>): string | null {
+  const subtype = asString(parsed.subtype, "");
+  const resultText = asString(parsed.result, "").trim();
+  const errors = extractClaudeErrorMessages(parsed);
+
+  let detail = resultText;
+  if (!detail && errors.length > 0) {
+    detail = errors[0] ?? "";
+  }
+
+  const parts = ["Claude run failed"];
+  if (subtype) parts.push(`subtype=${subtype}`);
+  if (detail) parts.push(detail);
+  return parts.length > 1 ? parts.join(": ") : null;
+}
+
+function extractClaudeErrorMessages(parsed: Record<string, unknown>): string[] {
+  const raw = Array.isArray(parsed.errors) ? parsed.errors : [];
+  const messages: string[] = [];
+
+  for (const entry of raw) {
+    if (typeof entry === "string") {
+      const msg = entry.trim();
+      if (msg) messages.push(msg);
+      continue;
+    }
+
+    if (typeof entry !== "object" || entry === null || Array.isArray(entry)) {
+      continue;
+    }
+
+    const obj = entry as Record<string, unknown>;
+    const msg = asString(obj.message, "") || asString(obj.error, "") || asString(obj.code, "");
+    if (msg) {
+      messages.push(msg);
+      continue;
+    }
+
+    try {
+      messages.push(JSON.stringify(obj));
+    } catch {
+      // skip non-serializable entry
+    }
+  }
+
+  return messages;
+}
+
+function isClaudeUnknownSessionError(parsed: Record<string, unknown>): boolean {
+  const resultText = asString(parsed.result, "").trim();
+  const allMessages = [resultText, ...extractClaudeErrorMessages(parsed)]
+    .map((msg) => msg.trim())
+    .filter(Boolean);
+
+  return allMessages.some((msg) =>
+    /no conversation found with session id|unknown session|session .* not found/i.test(msg),
+  );
+}
+
 function parseClaudeStreamJson(stdout: string) {
   let sessionId: string | null = null;
   let model = "";
@@ -623,7 +683,7 @@ export function heartbeatService(db: Db) {
       .update(agentRuntimeState)
       .set({
         adapterType: agent.adapterType,
-        sessionId: result.sessionId ?? existing.sessionId,
+        sessionId: result.clearSession ? null : (result.sessionId ?? existing.sessionId),
         lastRunId: run.id,
         lastRunStatus: run.status,
         lastError: result.errorMessage ?? null,
@@ -819,13 +879,34 @@ export function heartbeatService(db: Db) {
       context,
     });
 
+    const buildClaudeArgs = (resumeSessionId: string | null) => {
       const args = ["--print", prompt, "--output-format", "stream-json", "--verbose"];
-    if (sessionId) args.push("--resume", sessionId);
+      if (resumeSessionId) args.push("--resume", resumeSessionId);
       if (dangerouslySkipPermissions) args.push("--dangerously-skip-permissions");
       if (model) args.push("--model", model);
       if (maxTurns > 0) args.push("--max-turns", String(maxTurns));
       if (extraArgs.length > 0) args.push(...extraArgs);
+      return args;
+    };
 
+    const parseFallbackErrorMessage = (proc: RunProcessResult) => {
+      const stderrLine =
+        proc.stderr
+          .split(/\r?\n/)
+          .map((line) => line.trim())
+          .find(Boolean) ?? "";
+
+      if ((proc.exitCode ?? 0) === 0) {
+        return "Failed to parse claude JSON output";
+      }
+
+      return stderrLine
+        ? `Claude exited with code ${proc.exitCode ?? -1}: ${stderrLine}`
+        : `Claude exited with code ${proc.exitCode ?? -1}`;
+    };
+
+    const runAttempt = async (resumeSessionId: string | null) => {
+      const args = buildClaudeArgs(resumeSessionId);
       if (onMeta) {
         await onMeta({
           adapterType: "claude_local",
@@ -846,30 +927,41 @@ export function heartbeatService(db: Db) {
         onLog,
       });
 
+      const parsedStream = parseClaudeStreamJson(proc.stdout);
+      const parsed = parsedStream.resultJson ?? parseJson(proc.stdout);
+      return { proc, parsedStream, parsed };
+    };
+
+    const toAdapterResult = (
+      attempt: {
+        proc: RunProcessResult;
+        parsedStream: ReturnType<typeof parseClaudeStreamJson>;
+        parsed: Record<string, unknown> | null;
+      },
+      opts: { fallbackSessionId: string | null; clearSessionOnMissingSession?: boolean },
+    ): AdapterExecutionResult => {
+      const { proc, parsedStream, parsed } = attempt;
       if (proc.timedOut) {
         return {
           exitCode: proc.exitCode,
           signal: proc.signal,
           timedOut: true,
           errorMessage: `Timed out after ${timeoutSec}s`,
+          clearSession: Boolean(opts.clearSessionOnMissingSession),
         };
       }
 
-    const parsedStream = parseClaudeStreamJson(proc.stdout);
-    const parsed = parsedStream.resultJson ?? parseJson(proc.stdout);
       if (!parsed) {
         return {
           exitCode: proc.exitCode,
           signal: proc.signal,
           timedOut: false,
-        errorMessage:
+          errorMessage: parseFallbackErrorMessage(proc),
-          (proc.exitCode ?? 0) === 0
-            ? "Failed to parse claude JSON output"
-            : `Claude exited with code ${proc.exitCode ?? -1}`,
           resultJson: {
             stdout: proc.stdout,
             stderr: proc.stderr,
           },
+          clearSession: Boolean(opts.clearSessionOnMissingSession),
         };
       }
 
@@ -884,21 +976,46 @@ export function heartbeatService(db: Db) {
           };
         })();
 
+      const resolvedSessionId =
+        parsedStream.sessionId ??
+        (asString(parsed.session_id, opts.fallbackSessionId ?? "") || opts.fallbackSessionId);
+
       return {
         exitCode: proc.exitCode,
         signal: proc.signal,
         timedOut: false,
-      errorMessage: (proc.exitCode ?? 0) === 0 ? null : `Claude exited with code ${proc.exitCode ?? -1}`,
+        errorMessage:
+          (proc.exitCode ?? 0) === 0
+            ? null
+            : describeClaudeFailure(parsed) ?? `Claude exited with code ${proc.exitCode ?? -1}`,
         usage,
-      sessionId:
+        sessionId: resolvedSessionId,
-        parsedStream.sessionId ??
-        (asString(parsed.session_id, runtime.sessionId ?? "") || runtime.sessionId),
         provider: "anthropic",
         model: parsedStream.model || asString(parsed.model, model),
         costUsd: parsedStream.costUsd ?? asNumber(parsed.total_cost_usd, 0),
         resultJson: parsed,
         summary: parsedStream.summary || asString(parsed.result, ""),
+        clearSession: Boolean(opts.clearSessionOnMissingSession && !resolvedSessionId),
       };
+    };
+
+    const initial = await runAttempt(sessionId ?? null);
+    if (
+      sessionId &&
+      !initial.proc.timedOut &&
+      (initial.proc.exitCode ?? 0) !== 0 &&
+      initial.parsed &&
+      isClaudeUnknownSessionError(initial.parsed)
+    ) {
+      await onLog(
+        "stderr",
+        `[paperclip] Claude resume session "${sessionId}" is unavailable; retrying with a fresh session.\n`,
+      );
+      const retry = await runAttempt(null);
+      return toAdapterResult(retry, { fallbackSessionId: null, clearSessionOnMissingSession: true });
+    }
+
+    return toAdapterResult(initial, { fallbackSessionId: runtime.sessionId });
   }
 
   async function executeCodexLocalRun(

ui/src/pages/AgentDetail.tsx

@@ -110,7 +110,7 @@ type TranscriptEntry =
   | { kind: "assistant"; ts: string; text: string }
   | { kind: "tool_call"; ts: string; name: string; input: unknown }
   | { kind: "init"; ts: string; model: string; sessionId: string }
-  | { kind: "result"; ts: string; text: string; inputTokens: number; outputTokens: number; cachedTokens: number; costUsd: number }
+  | { kind: "result"; ts: string; text: string; inputTokens: number; outputTokens: number; cachedTokens: number; costUsd: number; subtype: string; isError: boolean; errors: string[] }
   | { kind: "stderr"; ts: string; text: string }
   | { kind: "system"; ts: string; text: string }
   | { kind: "stdout"; ts: string; text: string };
@@ -124,6 +124,23 @@ function asNumber(value: unknown): number {
   return typeof value === "number" && Number.isFinite(value) ? value : 0;
 }
 
+function errorText(value: unknown): string {
+  if (typeof value === "string") return value;
+  const rec = asRecord(value);
+  if (!rec) return "";
+  const msg =
+    (typeof rec.message === "string" && rec.message) ||
+    (typeof rec.error === "string" && rec.error) ||
+    (typeof rec.code === "string" && rec.code) ||
+    "";
+  if (msg) return msg;
+  try {
+    return JSON.stringify(rec);
+  } catch {
+    return "";
+  }
+}
+
 function parseClaudeStdoutLine(line: string, ts: string): TranscriptEntry[] {
   const parsed = asRecord(safeJsonParse(line));
   if (!parsed) {
@@ -171,6 +188,9 @@ function parseClaudeStdoutLine(line: string, ts: string): TranscriptEntry[] {
     const outputTokens = asNumber(usage.output_tokens);
     const cachedTokens = asNumber(usage.cache_read_input_tokens);
     const costUsd = asNumber(parsed.total_cost_usd);
+    const subtype = typeof parsed.subtype === "string" ? parsed.subtype : "";
+    const isError = parsed.is_error === true;
+    const errors = Array.isArray(parsed.errors) ? parsed.errors.map(errorText).filter(Boolean) : [];
     const text = typeof parsed.result === "string" ? parsed.result : "";
     return [{
       kind: "result",
@@ -180,6 +200,9 @@ function parseClaudeStdoutLine(line: string, ts: string): TranscriptEntry[] {
       outputTokens,
       cachedTokens,
       costUsd,
+      subtype,
+      isError,
+      errors,
     }];
   }
 
@@ -701,12 +724,12 @@ function RunsTab({ runs, companyId, agentId, selectedRunId }: { runs: HeartbeatR
   const selectedRun = sorted.find((r) => r.id === effectiveRunId) ?? null;
 
   return (
-    <div className="flex gap-0 border border-border rounded-lg overflow-hidden" style={{ height: "calc(100vh - 220px)" }}>
+    <div className="flex gap-0">
-      {/* Left: run list */}
+      {/* Left: run list — sticky, scrolls independently */}
       <div className={cn(
-        "shrink-0 overflow-y-auto border-r border-border",
+        "shrink-0 border border-border rounded-lg overflow-y-auto sticky top-4 self-start",
         selectedRun ? "w-72" : "w-full",
-      )}>
+      )} style={{ maxHeight: "calc(100vh - 2rem)" }}>
         {sorted.map((run) => {
           const statusInfo = runStatusIcons[run.status] ?? { icon: Clock, color: "text-neutral-400" };
           const StatusIcon = statusInfo.icon;
@@ -759,9 +782,9 @@ function RunsTab({ runs, companyId, agentId, selectedRunId }: { runs: HeartbeatR
         })}
       </div>
 
-      {/* Right: run detail */}
+      {/* Right: run detail — natural height, page scrolls */}
       {selectedRun && (
-        <div className="flex-1 min-w-0 overflow-y-auto pr-2">
+        <div className="flex-1 min-w-0 pl-4">
           <RunDetail key={selectedRun.id} run={selectedRun} />
         </div>
       )}
@@ -1165,7 +1188,7 @@ function LogViewer({ run }: { run: HeartbeatRun }) {
           </span>
         )}
       </div>
-      <div className="bg-neutral-950 rounded-lg p-3 font-mono text-xs max-h-80 overflow-y-auto space-y-0.5">
+      <div className="bg-neutral-950 rounded-lg p-3 font-mono text-xs space-y-0.5" style={{ maxHeight: "200vh" }}>
         {transcript.length === 0 && !run.logRef && (
           <div className="text-neutral-500">No persisted transcript for this run.</div>
         )}
@@ -1218,6 +1241,12 @@ function LogViewer({ run }: { run: HeartbeatRun }) {
                     tokens in={formatTokens(entry.inputTokens)} out={formatTokens(entry.outputTokens)} cached={formatTokens(entry.cachedTokens)} cost=${entry.costUsd.toFixed(6)}
                   </span>
                 </div>
+                {(entry.subtype || entry.isError || entry.errors.length > 0) && (
+                  <div className="ml-[74px] text-red-300 whitespace-pre-wrap break-all">
+                    subtype={entry.subtype || "unknown"} is_error={entry.isError ? "true" : "false"}
+                    {entry.errors.length > 0 ? ` errors=${entry.errors.join(" | ")}` : ""}
+                  </div>
+                )}
                 {entry.text && (
                   <div className="ml-[74px] whitespace-pre-wrap break-all text-neutral-100">{entry.text}</div>
                 )}
@@ -1252,10 +1281,46 @@ function LogViewer({ run }: { run: HeartbeatRun }) {
         <div ref={logEndRef} />
       </div>
 
+      {(run.status === "failed" || run.status === "timed_out") && (
+        <div className="rounded-lg border border-red-500/30 bg-red-950/20 p-3 space-y-2">
+          <div className="text-xs font-medium text-red-300">Failure details</div>
+          {run.error && (
+            <div className="text-xs text-red-200">
+              <span className="text-red-300">Error: </span>
+              {run.error}
+            </div>
+          )}
+          {run.stderrExcerpt && run.stderrExcerpt.trim() && (
+            <div>
+              <div className="text-xs text-red-300 mb-1">stderr excerpt</div>
+              <pre className="bg-neutral-950 rounded-md p-2 text-xs overflow-x-auto whitespace-pre-wrap text-red-100">
+                {run.stderrExcerpt}
+              </pre>
+            </div>
+          )}
+          {run.resultJson && (
+            <div>
+              <div className="text-xs text-red-300 mb-1">adapter result JSON</div>
+              <pre className="bg-neutral-950 rounded-md p-2 text-xs overflow-x-auto whitespace-pre-wrap text-red-100">
+                {JSON.stringify(run.resultJson, null, 2)}
+              </pre>
+            </div>
+          )}
+          {run.stdoutExcerpt && run.stdoutExcerpt.trim() && !run.resultJson && (
+            <div>
+              <div className="text-xs text-red-300 mb-1">stdout excerpt</div>
+              <pre className="bg-neutral-950 rounded-md p-2 text-xs overflow-x-auto whitespace-pre-wrap text-red-100">
+                {run.stdoutExcerpt}
+              </pre>
+            </div>
+          )}
+        </div>
+      )}
+
       {events.length > 0 && (
         <div>
           <div className="mb-2 text-xs font-medium text-muted-foreground">Events ({events.length})</div>
-          <div className="bg-neutral-950 rounded-lg p-3 font-mono text-xs max-h-56 overflow-y-auto space-y-0.5">
+          <div className="bg-neutral-950 rounded-lg p-3 font-mono text-xs space-y-0.5" style={{ maxHeight: "100vh" }}>
             {events.map((evt) => {
               const color = evt.color
                 ?? (evt.level ? levelColors[evt.level] : null)