[rig-decomposition-bench] Daily rig decomposition benchmark — 2026-10-20 — no result (task picker returned null) #684

import { agent, workflow, s, repair, configureAgent, copilotEngine } from "rig";

configureAgent(copilotEngine());

const DEADLINE = Date.now() + 25 * 60_000;
const left = () => DEADLINE - Date.now();

type Run = { code: number | null; stdout: string; stderr: string; timedOut: boolean };

async function runCli(source: string, args: string[], timeoutMs: number): Promise<Run> {
  const { spawn } = await import("node:child_process");
  return new Promise<Run>((resolve) => {
    let stdout = "";
    let stderr = "";
    let timedOut = false;
    const child = spawn("node", ["rig.ts", ...args], { cwd: process.cwd(), stdio: ["pipe", "pipe", "pipe"] });
    const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); }, Math.max(1000, timeoutMs));
    child.stdout.on("data", (d: Buffer) => { stdout += d.toString(); });
    child.stderr.on("data", (d: Buffer) => { stderr += d.toString(); });
    child.on("error", (e: Error) => { clearTimeout(timer); resolve({ code: -1, stdout, stderr: stderr + String(e), timedOut }); });
    child.on("close", (code: number | null) => { clearTimeout(timer); resolve({ code, stdout, stderr, timedOut }); });
    child.stdin.on("error", () => {});
    child.stdin.end(source);
  });
}

const cap = (t: string, n = 6000) => (t.length > n ? t.slice(0, n) + "\n...[truncated]" : t);

// Agent role: pick a concrete, gradeable, complicated daily task suited to multi-agent decomposition.
const pickTask = agent({
  name: "pickTask",
  model: "small",
  maxTurns: 2,
  addons: [repair()],
  instructions:
    "Pick one concrete, complicated task a person or team might face in a single day that would naturally benefit from being split across sub-agents running different models: multi-source research and synthesis, a multi-file coding task with design/implementation/review phases, a structured report combining several independent analyses, or a multi-step data transformation pipeline. It must be self-contained (solvable from its description alone, no external files or live web access) and concrete enough to grade. Prefer variety across domains; choose a random domain. Return a title, domain, description, and 3-6 concrete checkable success criteria.",
  output: s.object({
    title: s.string,
    domain: s.string,
    description: s.string,
    criteria: s.array(s.string),
  }),
});

// Agent role: solve the entire task alone in a single turn.
const solveSingle = agent({
  name: "solveSingle",
  model: "medium",
  maxTurns: 1,
  instructions:
    "Solve the whole task yourself in one response, addressing every success criterion. No delegation, no tools. Return the complete solution.",
  output: s.string,
});

// Agent role: write a decomposed rig program for the task.
const writeProgram = agent({
  name: "writeProgram",
  model: "medium",
  maxTurns: 1,
  instructions:
    "You write rig TypeScript programs. Output ONLY the complete program source (no prose; a single ts code fence is allowed). Rules: import only from \"rig\"; split the given task across at least two agents, each with a `// Agent role: ...` comment above it, choosing model \"small\" for simple sub-steps and \"medium\" or \"large\" for harder ones; coordinate them so the final answer combines their outputs rather than one agent solving everything; the root takes no required input and is exported with `export default` and never invoked. A workflow() spec accepts only `meta`, an optional `input`, and `body`; it has no `output` or `execute` field, and the final { solution } object is simply what body returns. An agent() root instead declares `output: s.object({ solution: s.string })`. Use `output: s.string` for every agent whose result is source text or free-form solution; do not make agents re-emit large solutions in object wrappers. Required shape (abridged to one agent; you need at least two):\n\n" +
    "```ts\nimport { agent, workflow, s } from \"rig\";\n\n// Agent role: <role>\nconst analyze = agent({ model: \"small\", instructions: \"...\", output: s.string });\n\n// Workflow role: combines the agents' outputs into the final solution\nexport default workflow({\n  meta: { name: \"decomposed-solution\", description: \"...\" },\n  body: async ({ call }) => {\n    const analysis = await call(analyze, \"...\");\n    return { solution: analysis ?? \"\" };\n  },\n});\n```\n\nThe `call(agent, promptString)` form passes a string input; agents may use `call(worker, input, { label })`. Agent results may be null on failure; handle with `?? \"\"`.",
  output: s.string,
});

// Agent role: fix a failing rig program, or write one from scratch when no source exists.
const fixProgram = agent({
  name: "fixProgram",
  model: "medium",
  maxTurns: 1,
  instructions:
    "You fix rig TypeScript programs. Output ONLY the complete corrected program source (no prose; one ts code fence allowed). Fix precisely what the error names and preserve what already worked. Constraints: import only from \"rig\"; at least two agents each with `// Agent role: ...` comment, models small/medium/large; final answer combines their outputs; root takes no input and is `export default`ed, never invoked. workflow() accepts only meta, optional input, and body (no output/execute); body returns { solution }. Use `output: s.string` for free-form agents. Required shape (abridged to one agent):\n\n" +
    "```ts\nimport { agent, workflow, s } from \"rig\";\n\n// Agent role: <role>\nconst analyze = agent({ model: \"small\", instructions: \"...\", output: s.string });\n\n// Workflow role: combines the agents' outputs into the final solution\nexport default workflow({\n  meta: { name: \"decomposed-solution\", description: \"...\" },\n  body: async ({ call }) => {\n    const analysis = await call(analyze, \"...\");\n    return { solution: analysis ?? \"\" };\n  },\n});\n```",
  output: s.string,
});

// Agent role: grade both solutions on correctness and completeness.
const grade = agent({
  name: "grade",
  model: "large",
  maxTurns: 1,
  addons: [repair()],
  instructions:
    "Score each solution 0-10 on how completely and correctly it satisfies the success criteria. Judge only correctness and completeness of content, not length and not which approach produced it. Pick winner single-call, decomposed, or tie, and explain the comparison.",
  output: s.object({
    singleScore: s.number,
    decomposedScore: s.number,
    winner: s.enum("single-call", "decomposed", "tie"),
    rationale: s.string,
  }),
});

function stripFence(t: string): string {
  const m = t.match(/```(?:ts|typescript)?\s*\n([\s\S]*?)```/);
  return (m ? m[1] : t).trim();
}

type Attempt = {
  attempt: number;
  generation: string;
  typecheck: string;
  execute: string;
  error: string;
  note: string;
};

// Workflow role: pick a task, solve it two ways, run the generated program, and grade.
export default workflow({
  meta: { name: "decomposition-bench", description: "Single-call vs decomposed benchmark", phases: ["Pick", "Single", "Decompose", "Grade"] },
  body: async ({ call, phase, log }) => {
    phase("Pick");
    const task = await call(pickTask, "Pick the task now.", { label: "pick" });
    if (!task) {
      return { status: "fail", reason: "task picker returned null", task: null, attempts: [] as Attempt[], source: "" };
    }
    const taskText = `Title: ${task.title}\nDomain: ${task.domain}\nDescription: ${task.description}\nSuccess criteria:\n${task.criteria.map((c: string) => `- ${c}`).join("\n")}`;

    phase("Single");
    const t0 = Date.now();
    const single = (await call(solveSingle, taskText, { label: "single", timeout: Math.max(5000, Math.min(left(), 6 * 60_000)) })) ?? "";
    const singleMs = Date.now() - t0;

    phase("Decompose");
    const d0 = Date.now();
    const attempts: Attempt[] = [];
    let source = "";
    let decomposed = "";
    let status = "fail";
    let lastError = "";
    for (let n = 1; n <= 2; n++) {
      if (left() < 4 * 60_000) { log("insufficient budget for another attempt"); break; }
      const rec: Attempt = { attempt: n, generation: "", typecheck: "skipped", execute: "skipped", error: "", note: "" };
      attempts.push(rec);
      let out: string | null;
      const wt = Math.max(5000, Math.min(left() - 60_000, 6 * 60_000));
      if (n === 1) {
        out = await call(writeProgram, taskText, { label: "write", timeout: wt });
      } else if (source.trim()) {
        out = await call(fixProgram, `Task:\n${taskText}\n\nCurrent source:\n${source}\n\nExact captured error:\n${lastError}\n\nFix the program.`, { label: "fix", timeout: wt });
      } else {
        out = await call(fixProgram, `There is no source to fix: the previous attempt produced no program. Generate the complete program from scratch for this task.\n\nTask:\n${taskText}`, { label: "fix-scratch", timeout: wt });
      }
      const src = out ? stripFence(out) : "";
      if (!src.trim()) {
        rec.generation = "empty (null or whitespace-only)";
        rec.error = "source generation returned no usable source";
        lastError = rec.error;
        if (n === 1) rec.note = "second attempt asks the fixer to generate from scratch";
        continue;
      }
      rec.generation = `ok (${src.length} chars)`;
      source = src;
      if (left() < 60_000) { rec.error = "time budget exhausted before typecheck"; break; }
      const tc = await runCli(src, ["--typecheck"], Math.min(2 * 60_000, left()));
      if (tc.code !== 0) {
        rec.typecheck = tc.timedOut ? "timeout" : "fail";
        rec.error = cap(tc.stdout + tc.stderr);
        lastError = rec.error;
        if (n === 1) rec.note = "fixer given source and the typecheck error";
        continue;
      }
      rec.typecheck = "pass";
      if (left() < 30_000) { rec.error = "time budget exhausted before execute"; break; }
      const ex = await runCli(src, ["--server"], Math.min(5 * 60_000, left()));
      if (ex.code !== 0 || !ex.stdout.trim()) {
        rec.execute = ex.timedOut ? "timeout" : "fail";
        rec.error = cap(ex.stderr + (ex.stdout ? "\n[stdout]\n" + ex.stdout : "") || "no output");
        lastError = rec.error;
        if (n === 1) rec.note = "fixer given source and the execute error";
        continue;
      }
      rec.execute = "pass";
      try {
        const parsed = JSON.parse(ex.stdout);
        decomposed = typeof parsed === "string" ? parsed : typeof parsed?.solution === "string" ? parsed.solution : JSON.stringify(parsed);
      } catch {
        decomposed = ex.stdout.trim();
      }
      if (decomposed.trim()) status = "pass";
      else rec.error = "execution succeeded but solution was empty";
      break;
    }
    const decomposedMs = Date.now() - d0;

    phase("Grade");
    const placeholder = "[Decomposition produced no solution: all attempts failed or the time budget was exhausted.]";
    const graded = left() > 30_000
      ? await call(grade, `${taskText}\n\n=== SOLUTION A (single-call) ===\n${single || "[empty]"}\n\n=== SOLUTION B (decomposed) ===\n${decomposed.trim() ? decomposed : placeholder}`, { label: "grade", timeout: Math.min(4 * 60_000, left()) })
      : null;

    return { status, task, singleMs, decomposedMs, single, decomposed, attempts, source, grading: graded };
  },
});