[rig-decomposition-bench] Daily rig decomposition benchmark — 2026-10-19 — no result (task picker failed) #678

import { agent, workflow, s, repair, configureAgent, copilotEngine } from "rig";

configureAgent(copilotEngine());

const DEADLINE = Date.now() + 25 * 60_000;
const left = () => DEADLINE - Date.now();

const SKELETON = `import { agent, workflow, s } from "rig";

// Agent role: <role>
const analyze = agent({ model: "small", instructions: "...", output: s.string });

// Workflow role: combines the agents' outputs into the final solution
export default workflow({
  meta: { name: "decomposed-solution", description: "..." },
  body: async ({ call }) => {
    const analysis = await call(analyze, "...");
    return { solution: analysis ?? "" };
  },
});`;

const CONTRACT = `You write a self-contained TypeScript program for the "rig" library. Rules:
- Import ONLY from "rig" (agent, workflow, s). No other imports, no console.log.
- Split the task across AT LEAST TWO agents, each preceded by a "// Agent role: ..." comment. Each agent sets model: "small" for simple sub-steps, "medium" or "large" for harder ones.
- Coordinate them so the final answer combines the agents' outputs rather than one agent solving everything.
- Root must be exported with "export default" and must NOT be invoked. It takes no required input.
- A workflow() spec accepts only meta, an optional input, and body. It has NO output or execute field; the final { solution } object is simply what body returns. body receives { call } (also pipeline, parallel, phase, log); call(agent, input) returns the agent output or null. An agent() root instead declares output: s.object({ solution: s.string }).
- Use s.string as output for every agent whose result is source text or free-form solution. Do not make any agent re-emit a large solution inside an object wrapper.
- Required shape (abridged to one agent; you need at least two):
${SKELETON}
Return ONLY the complete TypeScript source, with no commentary.`;

// Agent role: pick a concrete benchmark task
const picker = agent({
  model: "small",
  maxTurns: 2,
  addons: [repair()],
  instructions: `Pick ONE concrete, complicated task a person or team might face in a single day that would naturally benefit from being split across sub-agents running different models: multi-source research and synthesis, a multi-file coding task with design/implementation/review phases, a structured report combining independent analyses, or a multi-step data transformation pipeline. It must be self-contained (solvable from its description alone, no external files or live web) and concrete enough to grade. Vary the domain; today is ${new Date().toISOString().slice(0, 10)} so pick something unusual. Return a title, domain, description, and 3-6 concrete checkable success criteria.`,
  output: s.object({
    title: s.string,
    domain: s.string,
    description: s.string,
    criteria: s.array(s.string),
  }),
});

// Agent role: solve the whole task in one turn
const solver = agent({
  model: "medium",
  maxTurns: 1,
  instructions: "Solve the whole task yourself in a single response, addressing every success criterion. No delegation, no tools. Return the complete solution.",
  output: s.string,
});

// Agent role: write or fix the decomposed rig program
const writer = agent({
  model: "medium",
  maxTurns: 1,
  instructions: CONTRACT,
  output: s.string,
});

// Agent role: grade both solutions
const grader = agent({
  model: "large",
  maxTurns: 1,
  instructions: "Score each solution 0-10 on how completely and correctly it satisfies the success criteria. Judge only correctness and completeness of content, not length and not which approach produced it. Pick winner single-call, decomposed, or tie, and explain the comparison.",
  output: s.object({
    singleScore: s.number,
    decomposedScore: s.number,
    winner: s.enum("single-call", "decomposed", "tie"),
    rationale: s.string,
  }),
});

type RunResult = { code: number | null; stdout: string; stderr: string; timedOut: boolean };

function runCli(args: string[], input: string, timeoutMs: number): Promise<RunResult> {
  const cp = process.getBuiltinModule("node:child_process");
  return new Promise((resolve) => {
    let stdout = "";
    let stderr = "";
    let timedOut = false;
    const child = cp.spawn(process.execPath, ["rig.ts", ...args], { cwd: process.cwd(), stdio: ["pipe", "pipe", "pipe"] });
    const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); }, timeoutMs);
    child.stdout.on("data", (d: Buffer) => { stdout += d.toString(); });
    child.stderr.on("data", (d: Buffer) => { stderr += d.toString(); });
    child.on("error", (e: Error) => { stderr += String(e); });
    child.on("close", (code: number | null) => { clearTimeout(timer); resolve({ code, stdout, stderr, timedOut }); });
    child.stdin.on("error", () => {});
    child.stdin.end(input);
  });
}

type Attempt = { attempt: number; generation: string; typecheck: string; execute: string; error: string; note: string };

function stripFence(src: string): string {
  const m = src.match(/```(?:ts|typescript|rig)?\s*\n([\s\S]*?)```/);
  return (m ? m[1]! : src).trim();
}

// Workflow role: pick a task, solve it two ways, grade, and return the full report
export default workflow({
  meta: { name: "rig-decomposition-bench", description: "Compare single-call vs decomposed solving", phases: ["Pick", "Single", "Decompose", "Grade"] },
  body: async ({ call, phase, log }) => {
    phase("Pick");
    const task = await call(picker, "Pick the task now.", { timeout: Math.max(1000, Math.min(180_000, left())) });
    if (!task) return { status: "failed: task picker returned null", task: null, attempts: [] as Attempt[] };
    const taskText = `Title: ${task.title}\nDomain: ${task.domain}\nDescription: ${task.description}\nSuccess criteria:\n${task.criteria.map((c: string) => `- ${c}`).join("\n")}`;

    phase("Single");
    const t0 = Date.now();
    const single = await call(solver, taskText, { timeout: Math.max(1000, Math.min(480_000, left())) });
    const singleMs = Date.now() - t0;

    phase("Decompose");
    const d0 = Date.now();
    const attempts: Attempt[] = [];
    let source = "";
    let decomposed = "";
    let status = "fail";
    let prevSource = "";
    let prevError = "";
    for (let n = 1; n <= 2; n++) {
      if (left() < 120_000) { log("budget exhausted"); break; }
      let prompt: string;
      if (n === 1) prompt = `Write the program for this task:\n${taskText}`;
      else if (prevSource.trim()) prompt = `Fix the program below. Fix precisely what the error names and preserve what already worked.\n\nTask:\n${taskText}\n\nRequired shape:\n${SKELETON}\n\nSource:\n${prevSource}\n\nError:\n${prevError}`;
      else prompt = `There is no source to fix: the previous attempt produced nothing. Generate the complete program from scratch.\n\nTask:\n${taskText}\n\nRequired shape:\n${SKELETON}`;
      const raw = await call(writer, prompt, { timeout: Math.max(1000, Math.min(300_000, left())) });
      const src = raw ? stripFence(raw) : "";
      const rec: Attempt = { attempt: n, generation: "", typecheck: "skipped", execute: "skipped", error: "", note: "" };
      attempts.push(rec);
      if (!src.trim()) {
        rec.generation = "empty/null";
        rec.error = "writer returned null or whitespace-only source";
        prevSource = ""; prevError = rec.error;
        rec.note = n < 2 ? "Next attempt generates the program from scratch (no source to fix)." : "";
        continue;
      }
      rec.generation = "non-empty";
      source = src;
      prevSource = src;
      const tc = await runCli(["--typecheck"], src, Math.max(1000, Math.min(120_000, left())));
      if (tc.code !== 0) {
        rec.typecheck = tc.timedOut ? "timeout" : "fail";
        rec.error = `${tc.stdout}\n${tc.stderr}`.trim();
        prevError = rec.error;
        rec.note = n < 2 ? "Next attempt passes source and typecheck error to the fixer." : "";
        continue;
      }
      rec.typecheck = "pass";
      if (left() < 30_000) { rec.execute = "skipped (budget)"; break; }
      const ex = await runCli(["--server"], src, Math.max(1000, Math.min(300_000, left())));
      if (ex.code !== 0 || ex.timedOut) {
        rec.execute = ex.timedOut ? "timeout" : "fail";
        rec.error = `exit=${ex.code}\n${ex.stdout}\n${ex.stderr}`.trim();
        prevError = rec.error;
        rec.note = n < 2 ? "Next attempt passes source and execution error to the fixer." : "";
        continue;
      }
      let parsed: { solution?: unknown } | null = null;
      try { parsed = JSON.parse(ex.stdout); } catch { parsed = null; }
      if (parsed && typeof parsed.solution === "string" && parsed.solution.trim()) {
        rec.execute = "pass";
        decomposed = parsed.solution;
        status = "pass";
        break;
      }
      rec.execute = "fail";
      rec.error = `stdout was not JSON with non-empty solution:\n${ex.stdout}\n${ex.stderr}`.trim();
      prevError = rec.error;
      rec.note = n < 2 ? "Next attempt passes source and the output error to the fixer." : "";
    }
    const decomposedMs = Date.now() - d0;

    phase("Grade");
    const placeholder = "[Decomposition produced no solution: the generated program failed to produce output.]";
    let grade = null;
    if (left() > 20_000) {
      grade = await call(grader, `${taskText}\n\n=== SOLUTION A (single-call) ===\n${single ?? "[no solution]"}\n\n=== SOLUTION B (decomposed) ===\n${decomposed.trim() ? decomposed : placeholder}`, { timeout: Math.max(1000, Math.min(300_000, left())) });
    }
    return { task, singleMs, decomposedMs, singleSolution: single ?? "", decomposedSolution: decomposed, attempts, status, grade, decomposedSource: source };
  },
});