Antigravity の Retry を本番品質で設計する — 冪等性・バックオフ・コスト上限・失敗トリアージ

// src/retry/backoff.ts
// 指数バックオフ + ジッター + 最大試行回数 + 最大待ち時間の組み合わせ。
// これを「再試行してよい」と判断したエラーにだけ適用する。
 
export type RetryOptions = {
  maxAttempts: number;
  baseDelayMs: number;
  maxDelayMs: number;
  shouldRetry: (err: unknown, attempt: number) => boolean;
  onRetry?: (err: unknown, attempt: number, delayMs: number) => void;
};
 
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
 
export async function retryWithBackoff<T>(
  fn: () => Promise<T>,
  opts: RetryOptions,
): Promise<T> {
  let lastError: unknown;
  for (let attempt = 1; attempt <= opts.maxAttempts; attempt++) {
    try {
      return await fn();
    } catch (err) {
      lastError = err;
      if (attempt === opts.maxAttempts || !opts.shouldRetry(err, attempt)) {
        throw err;
      }
      // full jitter: 0 〜 base*2^attempt の範囲でランダム化
      const expo = Math.min(opts.baseDelayMs * 2 ** attempt, opts.maxDelayMs);
      const delay = Math.floor(Math.random() * expo);
      opts.onRetry?.(err, attempt, delay);
      await sleep(delay);
    }
  }
  throw lastError;
}
 
// 使い方
await retryWithBackoff(
  () => fetch("https://api.example.com/data").then((r) => r.json()),
  {
    maxAttempts: 5,
    baseDelayMs: 500,
    maxDelayMs: 30_000,
    shouldRetry: (err) => {
      // 後述する「失敗トリアージ」でここを埋める
      return isTransientError(err);
    },
    onRetry: (err, attempt, delay) => {
      console.log(`retry attempt=${attempt} delay=${delay}ms error=${String(err)}`);
    },
  },
);

// retry-sim.mjs — 実時間を待たずに仮想時計でバックオフ戦略を比較する
// node retry-sim.mjs で実行できます（依存パッケージなし）
 
function mulberry32(a) {
  return function () {
    a |= 0; a = (a + 0x6d2b79f5) | 0;
    let t = Math.imul(a ^ (a >>> 15), 1 | a);
    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
  };
}
 
function simulate(strategy, seed) {
  const rnd = mulberry32(seed);
  const WORKERS = 40, CAPACITY = 8, REFILL_MS = 250;
  const BASE = 500, MAX_DELAY = 30_000, MAX_ATTEMPTS = 6;
 
  let tokens = CAPACITY, lastRefill = 0;
  // イベントキュー: 各ワーカーが「いつ叩きに行くか」を仮想時刻で持つ
  const queue = Array.from({ length: WORKERS }, (_, id) => ({ t: 0, id, attempt: 0 }));
  let attempts = 0, ok = 0, fail = 0;
  const completedAt = [];
 
  while (queue.length) {
    queue.sort((a, b) => a.t - b.t);
    const e = queue.shift();
 
    const refill = Math.floor((e.t - lastRefill) / REFILL_MS);
    if (refill > 0) {
      tokens = Math.min(CAPACITY, tokens + refill);
      lastRefill += refill * REFILL_MS;
    }
 
    e.attempt++; attempts++;
    if (tokens > 0) { tokens--; ok++; completedAt.push(e.t); continue; }
    if (e.attempt >= MAX_ATTEMPTS) { fail++; continue; }
 
    const exp = Math.min(MAX_DELAY, BASE * 2 ** (e.attempt - 1));
    const delay =
      strategy === "fixed" ? BASE
      : strategy === "exponential" ? exp
      : strategy === "equal-jitter" ? exp / 2 + rnd() * (exp / 2)
      : rnd() * exp; // full-jitter
    queue.push({ t: e.t + delay, id: e.id, attempt: e.attempt });
  }
 
  completedAt.sort((a, b) => a - b);
  return {
    attempts, ok, fail,
    median: completedAt[Math.floor(completedAt.length / 2)],
    p95: completedAt[Math.floor(completedAt.length * 0.95)],
  };
}
 
for (const s of ["fixed", "exponential", "equal-jitter", "full-jitter"]) {
  let A = 0, O = 0, F = 0, M = 0, P = 0;
  const RUNS = 200;
  for (let r = 0; r < RUNS; r++) {
    const x = simulate(s, 1000 + r);
    A += x.attempts; O += x.ok; F += x.fail; M += x.median; P += x.p95;
  }
  console.log(
    `${s.padEnd(14)} 試行=${(A / RUNS).toFixed(1)} 完了率=${((100 * O) / (O + F)).toFixed(1)}% ` +
    `中央値=${(M / RUNS / 1000).toFixed(2)}s p95=${(P / RUNS / 1000).toFixed(2)}s`,
  );
}

// src/retry/budget.ts
// タスク単位のコスト上限をエージェント実行ループに組み込む。
// LLM レスポンスから usage を取り出し、合計が超えたら BudgetExceeded を投げる。
 
export class BudgetExceeded extends Error {
  constructor(
    public readonly used: number,
    public readonly budget: number,
  ) {
    super(`budget exceeded: used=${used} budget=${budget}`);
  }
}
 
export class TokenBudget {
  private used = 0;
 
  constructor(
    private readonly budget: number,
    private readonly onWarn?: (used: number, budget: number) => void,
  ) {}
 
  track(usage: { prompt_tokens: number; completion_tokens: number }) {
    this.used += usage.prompt_tokens + usage.completion_tokens;
    if (this.used > this.budget) {
      throw new BudgetExceeded(this.used, this.budget);
    }
    // 80% 超えで警告（Slack 通知・メトリクス発火などに繋げる）
    if (this.used > this.budget * 0.8) {
      this.onWarn?.(this.used, this.budget);
    }
  }
 
  remaining(): number {
    return Math.max(0, this.budget - this.used);
  }
}
 
// 使い方（エージェント実行ループの内側）
const budget = new TokenBudget(200_000, (used, total) => {
  console.warn(`budget at 80%: ${used}/${total}`);
});
 
for (const step of plannedSteps) {
  try {
    const response = await callLLM(step.prompt);
    budget.track(response.usage);
    applyResult(response);
  } catch (err) {
    if (err instanceof BudgetExceeded) {
      await notifyHuman("budget exceeded, human review required", { err });
      await saveCheckpoint(step);
      throw err; // ループ全体を止める
    }
    throw err;
  }
}

// src/retry/triage.ts
// エラーを transient / permanent / ambiguous に分類するトリアージ関数。
// このロジックは実運用で必ず育てる。新種エラーが来るたびに分類を追加する。
 
export type ErrorClass = "transient" | "permanent" | "ambiguous";
 
export function classifyError(err: unknown): ErrorClass {
  if (!(err instanceof Error)) return "ambiguous";
  const msg = err.message.toLowerCase();
 
  // --- 認証・権限・入力エラーは再試行しない ---
  if (/unauthorized|forbidden|invalid_api_key|authentication/.test(msg)) {
    return "permanent";
  }
  if (/validation|invalid request|bad request|schema/.test(msg)) {
    return "permanent";
  }
  if (/model_not_found|unknown model/.test(msg)) {
    return "permanent";
  }
 
  // --- 一時的なネットワーク・レートリミットは再試行する ---
  if (/rate.?limit|too many requests|429/.test(msg)) {
    return "transient";
  }
  if (/timeout|econnreset|etimedout|network/.test(msg)) {
    return "transient";
  }
  if (/503|service unavailable|overloaded/.test(msg)) {
    return "transient";
  }
 
  // --- 曖昧なものは一度だけ再試行する ---
  if (/500|internal server|json|parse/.test(msg)) {
    return "ambiguous";
  }
 
  return "ambiguous";
}
 
// 使い方: retry ループの shouldRetry に組み込む
export function shouldRetryFromTriage(err: unknown, attempt: number): boolean {
  const klass = classifyError(err);
  if (klass === "permanent") return false;
  if (klass === "transient") return attempt < 5;
  if (klass === "ambiguous") return attempt < 2; // 一度だけ
  return false;
}

// src/retry/checkpoint.ts
// チェックポイントと Retry を組み合わせた「再開可能なエージェント」の最小実装。
// 保存先は JSON ファイル / KV / Firestore など、耐久性のあるストレージを使うこと。
 
import fs from "node:fs/promises";
 
type Step = { id: string; run: () => Promise<void> };
 
type Checkpoint = {
  completed: string[];
  updatedAt: string;
};
 
export async function runResumable(
  steps: Step[],
  checkpointPath: string,
): Promise<void> {
  let cp: Checkpoint;
  try {
    cp = JSON.parse(await fs.readFile(checkpointPath, "utf-8"));
  } catch {
    cp = { completed: [], updatedAt: new Date().toISOString() };
  }
 
  for (const step of steps) {
    if (cp.completed.includes(step.id)) {
      console.log(`skip already-completed step: ${step.id}`);
      continue;
    }
    try {
      await step.run();
      cp.completed.push(step.id);
      cp.updatedAt = new Date().toISOString();
      await fs.writeFile(checkpointPath, JSON.stringify(cp, null, 2));
    } catch (err) {
      // 失敗時もチェックポイントを保存し、次回はここから再開できるようにする
      await fs.writeFile(checkpointPath, JSON.stringify(cp, null, 2));
      throw err;
    }
  }
}

// src/agents/resilient-loop.ts
// 冪等性・バックオフ・コスト上限・失敗トリアージ・チェックポイントを 1 本のループに束ねる
 
import { IdempotencyGuard } from "../retry/idempotency";
import { retryWithBackoff } from "../retry/backoff";
import { TokenBudget } from "../retry/budget";
import { classifyError, shouldRetryFromTriage } from "../retry/triage";
import { runResumable } from "../retry/checkpoint";
 
const guard = new IdempotencyGuard();
const budget = new TokenBudget(200_000, (used, total) => {
  notifySlack(`トークン予算 80% 到達: ${used}/${total}`);
});
 
type TaskStep = { id: string; action: string; payload: unknown };
 
async function runStep(step: TaskStep) {
  return retryWithBackoff(
    // 副作用は必ず guard の内側に置く。バックオフより内側であることが重要
    () =>
      guard.run(step.action, step.payload, async () => {
        const response = await callLLM({ action: step.action, payload: step.payload });
        budget.track(response.usage);
        return response;
      }),
    {
      maxAttempts: 5,
      baseDelayMs: 500,
      maxDelayMs: 30_000,
      shouldRetry: (err, attempt) => {
        if (classifyError(err) === "permanent") return false;          // 認証・権限は即降参
        if (String(err).includes("budget exceeded")) return false;      // 予算超過も再試行しない
        return shouldRetryFromTriage(err, attempt);
      },
      onRetry: (err, attempt, delayMs) => {
        logger.warn({ step: step.id, attempt, delayMs, err: String(err) }, "retry");
      },
    },
  );
}
 
export async function runTask(steps: TaskStep[]) {
  const resumable = steps.map((s) => ({ id: s.id, run: async () => { await runStep(s); } }));
  await runResumable(resumable, `.agent/checkpoint-${Date.now()}.json`);
}