Routing /effort by Task Class in Antigravity CLI: Six Weeks of Measurements

Automating the routing

Classification and routing live in one script. It appends every run to a JSONL file so acceptance rates can be aggregated later.

#!/usr/bin/env node
/**
 * effort-route.mjs
 * Picks an effort level from the task description, launches the Antigravity CLI (agy),
 * and appends the outcome to a JSONL log.
 *
 *   node effort-route.mjs "Move AdMob initialization into the Application class"
 *   node effort-route.mjs --dry-run "Fix the button label"
 *   node effort-route.mjs --mark <id> accept
 */
import { spawn } from "node:child_process";
import { appendFile, readFile } from "node:fs/promises";
import { resolve, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { randomUUID } from "node:crypto";
 
const HERE = dirname(fileURLToPath(import.meta.url));
const POLICY_PATH = resolve(HERE, "effort-policy.json");
const LOG_PATH = resolve(HERE, "effort-runs.jsonl");
 
// Level names change between CLI versions, so they live in the policy file, not in code.
const DEFAULT_POLICY = {
  fallback: "medium",
  rules: [
    { class: "mechanical",  level: "low",    match: "(typo|wording|label|rename|imports?|format)" },
    { class: "localized",   level: "medium", match: "(bug|fix|regression|add a test|single file)" },
    { class: "structural",  level: "high",   match: "(design|split|migrat|refactor|architect|replace)" },
    { class: "exploratory", level: "high",   match: "(investigate|why|root cause|compare|decide)" }
  ]
};
 
async function loadPolicy() {
  try {
    return JSON.parse(await readFile(POLICY_PATH, "utf8"));
  } catch (err) {
    if (err.code === "ENOENT") return DEFAULT_POLICY;
    // Swallowing a malformed JSON here drops every task into the fallback level,
    // and you can go days without noticing "somehow everything is medium". Fail loudly.
    throw new Error(`Could not read effort-policy.json: ${err.message}`);
  }
}
 
function classify(task, policy) {
  for (const rule of policy.rules) {
    if (new RegExp(rule.match, "i").test(task)) {
      return { taskClass: rule.class, level: rule.level };
    }
  }
  return { taskClass: "unclassified", level: policy.fallback };
}
 
function runAgy(task, level) {
  // In non-interactive mode I put /effort on the first line of the prompt body.
  // If a flag ships later, only this args array needs to change.
  const prompt = `/effort ${level}\n${task}`;
  return new Promise((done) => {
    const started = Date.now();
    const child = spawn("agy", ["-p", prompt], { stdio: ["ignore", "inherit", "inherit"] });
    child.on("error", (err) =>
      done({ ok: false, code: null, ms: Date.now() - started, error: err.message })
    );
    child.on("close", (code) =>
      done({ ok: code === 0, code, ms: Date.now() - started })
    );
  });
}
 
async function main() {
  const argv = process.argv.slice(2);
 
  if (argv[0] === "--mark") {
    const [, id, verdict] = argv;
    if (!id || !["accept", "rework"].includes(verdict)) {
      console.error("usage: --mark <id> accept|rework");
      process.exit(2);
    }
    await appendFile(LOG_PATH, JSON.stringify({ type: "mark", id, verdict }) + "\n");
    return;
  }
 
  const dryRun = argv[0] === "--dry-run";
  const task = (dryRun ? argv.slice(1) : argv).join(" ").trim();
  if (!task) {
    console.error("Pass a task description");
    process.exit(2);
  }
 
  const policy = await loadPolicy();
  const { taskClass, level } = classify(task, policy);
  const id = randomUUID().slice(0, 8);
 
  console.error(`[effort-route] ${id} class=${taskClass} effort=${level}`);
  if (dryRun) return;
 
  const result = await runAgy(task, level);
  await appendFile(
    LOG_PATH,
    JSON.stringify({
      type: "run", id, taskClass, level,
      ms: result.ms, ok: result.ok, error: result.error ?? null,
      at: new Date().toISOString()
    }) + "\n"
  );
 
  console.error(`[effort-route] ${id} ${result.ok ? "ok" : "failed"} ${Math.round(result.ms / 1000)}s`);
  console.error(`[effort-route] verdict: node effort-route.mjs --mark ${id} accept|rework`);
  if (!result.ok) process.exit(1);
}
 
main().catch((err) => {
  console.error(`[effort-route] ${err.message}`);
  process.exit(1);
});

Splitting --mark out was deliberate. I can almost never judge a run the moment it finishes — I decide after reading the diff. Printing the ID and letting me mark it later fits how the work actually goes.

The aggregation side can stay plain.

#!/usr/bin/env node
/** effort-report.mjs — first-pass acceptance and latency per class and level */
import { readFile } from "node:fs/promises";
import { resolve, dirname } from "node:path";
import { fileURLToPath } from "node:url";
 
const LOG_PATH = resolve(dirname(fileURLToPath(import.meta.url)), "effort-runs.jsonl");
const lines = (await readFile(LOG_PATH, "utf8")).split("\n").filter(Boolean).map(JSON.parse);
 
const runs = new Map();
for (const e of lines) {
  if (e.type === "run") runs.set(e.id, { ...e, verdict: null });
  if (e.type === "mark" && runs.has(e.id)) runs.get(e.id).verdict = e.verdict;
}
 
const buckets = new Map();
for (const r of runs.values()) {
  if (!r.verdict) continue; // unjudged runs stay out of the denominator
  const key = `${r.taskClass}/${r.level}`;
  const b = buckets.get(key) ?? { n: 0, accepted: 0, ms: [] };
  b.n += 1;
  if (r.verdict === "accept") b.accepted += 1;
  b.ms.push(r.ms);
  buckets.set(key, b);
}
 
const p50 = (a) => [...a].sort((x, y) => x - y)[Math.floor(a.length / 2)] ?? 0;
for (const [key, b] of [...buckets].sort()) {
  const rate = ((b.accepted / b.n) * 100).toFixed(0);
  console.log(`${key.padEnd(24)} n=${String(b.n).padStart(3)}  first-pass ${rate}%  p50 ${Math.round(p50(b.ms) / 1000)}s`);
}

Excluding unjudged runs mattered more than I expected. For a while I was folding forgotten judgments into "rework", which pushed the acceptance rate more than ten points below reality.

Six weeks of numbers

Between mid-June and late July 2026 I accumulated 212 judged runs. This is one person's log on one person's workload, so treat it as a direction rather than a benchmark.

I left acceptance blank for exploratory work on purpose. Most of those runs never produce a diff, so they don't belong on the same ruler. Forcing everything onto one metric is how you end up optimizing the wrong thing.

I also ran a deliberate control: fourteen structural tasks at medium instead of high. First-pass acceptance dropped from 52% to 44%. That's where the extra reasoning earned its keep — mostly in whether a failing test got traced back to its cause or patched at the surface.

Where raising effort made things worse

The mechanical class went the opposite way from what I assumed.

For the two weeks before I started measuring, I ran everything at high. Re-running those logs through the same aggregation puts mechanical first-pass acceptance at 61% — twenty-three points below the 84% I saw after dropping to low. The p50 was 2 m 18 s, more than three times slower.

Reading the diffs back made the reason obvious. Asked to fix wording in one place, the agent would also align similar strings nearby, sweep unused imports, and tighten a type or two along the way.

None of those changes are bad in isolation. But I had opened the diff expecting to confirm a single line, and my review posture didn't scale to what arrived. So I'd send it back with "I see the intent, but not this time" — and the round trip cost more than the fix.

Give an agent room to explore and it will explore. Obvious in hindsight. What I hadn't registered was that granting that room is now a decision I get to make per task.

When the correct answer is known in advance, don't hand over room to explore. That's the single most useful thing the measurements gave me.

What mattered more than effort

One factor produced a bigger gap than any level change.

Of the structural tasks, twenty-one included two or three lines of acceptance criteria in the request — things like "all existing tests must pass" and "don't change the public API signature". A definition of done, essentially.

Those twenty-one came in at 73% first-pass acceptance. The seventeen without criteria came in at 41%. A thirty-two point spread.

Dropping from high to medium moved the needle eight points. How I wrote the request mattered roughly four times more.

/effort is useful, but it is not a mechanism for resolving ambiguity. Raise the level on a vague request and you get a vague thing built carefully — and because it was built carefully, there's more of it to read.

My request template now always carries a line labeled "conditions for done". When I catch myself about to send it empty, that's a signal the task isn't articulated yet.

Putting the default in the agent definition

CLI v1.1.6, released 24 July 2026, allows custom agents to be defined in Markdown files. Storing a default level per role cuts down on maintaining the same decision in two places.

This is roughly the shape I use.

---
name: mechanical-fixer
description: Handles only changes whose correct answer is known in advance
effort: low
---
 
## In scope
- Wording and terminology fixes
- Renames, import tidying
 
## Out of scope
- Anything requiring a design judgment
- Cleanup of adjacent code that wasn't requested
 
## Conditions for done
- No files touched outside the requested change
- Existing tests pass

The "out of scope" section did the heavy lifting. When I only described the role, there was still room for adjacent cleanup to slip in. Writing down what not to do draws a sharper boundary than describing what to do.

These definitions are plain text, so they go straight into version control. I commit changes to them, which lets me line up a shift in acceptance rate against a change in the definition. Did I move the level, or did I reword the instructions? Keeping those separable is worth the small discipline.

Whether the CLI honors an effort key in the front matter depends on your version. I confirmed it was picked up in the startup log before relying on it. If yours doesn't, keeping the level in the router's policy file gets you the same result.

Rolling it out in order

Changing everything at once makes it impossible to tell what worked. This is the order I used.

1. Log for a week without changing anything — run effort-route.mjs with --dry-run so it records the classification and your current level, and nothing else
2. Build the judging habit — after reviewing a diff, run --mark <id> accept|rework. If this doesn't stick, no numbers follow
3. Drop only mechanical to low — start with the class that's easiest to read and easiest to revert
4. Aggregate two weeks and compare — line up first-pass acceptance and p50 before and after, and check nothing regressed
5. Add a "conditions for done" line to your request template — get this habit in place before touching levels
6. Move defaults into the agent definition — collapsing the router and the definition file into one source can wait until last

For picking a level, here's how I think about it now.

Closing

Three lines, if I fold the six weeks down:

Don't hand over room to explore when the answer is known in advance. Do hand it over when the blast radius isn't legible. And before touching the level at all, write one line describing what "done" means.

If you want to try this, start with the reporting script rather than the router. Routing rules depend heavily on the shape of your own work, and until you can see which levels you're currently using and how often you're sending things back, there's no way to tell which rules are right. A single week of logging will surface a skew you didn't know you had.

I'm still adjusting — I haven't settled on a fair ruler for exploratory work. If you've drawn that line somewhere that works, I'd be glad to hear where.