AGENTS.md を一行直したら挙動が崩れた — 指示そのものを回帰テストする仕組み

# fixtures/reuse-existing-util.yaml
id: reuse-existing-util
prompt: |
  src/utils.py に slugify があります。
  記事タイトルから slug を作る処理を追加してください。
setup: |
  mkdir -p src
  printf 'def slugify(s):\n    return s.lower().replace(" ", "-")\n' > src/utils.py
assertions:
  - type: file_exists
    path: src/utils.py
  - type: grep
    path: src
    pattern: "from .utils import slugify|utils.slugify|slugify\\("
    must: present
  - type: grep
    path: src
    pattern: "def slugify"
    count_max: 1        # slugify を再定義していないこと
  - type: forbidden
    path: src
    pattern: "re.sub"    # 既存を無視して正規表現で書き直していないか

# harness.py
import subprocess, tempfile, shutil, glob, re, os, sys, json
from pathlib import Path
import yaml
 
def run_agent(workdir: str, prompt: str) -> int:
    """Antigravity CLI をヘッドレスで実行する差し込み口。
    実運用では下の1行を CLI 呼び出しに置き換える。"""
    cmd = ["antigravity", "run", "--headless", "--yes", "-p", prompt]
    proc = subprocess.run(cmd, cwd=workdir, capture_output=True, text=True, timeout=120)
    return proc.returncode
 
def _files(base: str, path: str):
    root = os.path.join(base, path)
    if os.path.isfile(root):
        return [root]
    return [p for p in glob.glob(os.path.join(root, "**", "*"), recursive=True)
            if os.path.isfile(p)]
 
def check(base: str, a: dict) -> bool:
    t = a["type"]
    if t == "file_exists":
        return os.path.isfile(os.path.join(base, a["path"]))
    if t in ("grep", "forbidden"):
        pat = re.compile(a["pattern"])
        hits = 0
        for f in _files(base, a["path"]):
            try:
                hits += len(pat.findall(Path(f).read_text(errors="ignore")))
            except OSError:
                pass
        if t == "forbidden":
            return hits == 0
        if a.get("must") == "present" and hits == 0:
            return False
        if "count_max" in a and hits > a["count_max"]:
            return False
        return True
    if t == "exit_zero":
        r = subprocess.run(a["cmd"], cwd=base, shell=True, capture_output=True)
        return r.returncode == 0
    raise ValueError(f"unknown assertion: {t}")
 
def run_fixture(fx: dict, trials: int) -> dict:
    passes = 0
    for _ in range(trials):
        base = tempfile.mkdtemp(prefix="fx-")
        try:
            if fx.get("setup"):
                subprocess.run(fx["setup"], cwd=base, shell=True, check=False)
            run_agent(base, fx["prompt"])
            ok = all(check(base, a) for a in fx["assertions"])
            passes += 1 if ok else 0
        finally:
            shutil.rmtree(base, ignore_errors=True)
    return {"id": fx["id"], "passes": passes, "trials": trials,
            "score": round(passes / trials, 3)}
 
if __name__ == "__main__":
    trials = int(os.environ.get("TRIALS", "3"))
    results = []
    for path in sorted(glob.glob("fixtures/*.yaml")):
        fx = yaml.safe_load(Path(path).read_text())
        results.append(run_fixture(fx, trials))
    json.dump({"results": results}, sys.stdout, ensure_ascii=False, indent=2)

# compare.py — 変更前後のスコアを比較して退行を検知する
import json, sys
 
def load(p):
    return {r["id"]: r["score"] for r in json.load(open(p))["results"]}
 
base = load(sys.argv[1])     # baseline.json（変更前）
head = load(sys.argv[2])     # current.json（変更後）
TOL = 0.34                   # 3試行なら1回落ちる程度は許容
 
regressed = []
for fid, b in base.items():
    h = head.get(fid, 0.0)
    if b - h > TOL:
        regressed.append((fid, b, h))
 
for fid, b, h in regressed:
    print(f"REGRESSION {fid}: {b:.2f} -> {h:.2f}")
 
sys.exit(1 if regressed else 0)

#!/usr/bin/env bash
# ci-regression.sh — 指示文の変更で退行が出たら止める
set -euo pipefail
TRIALS=3 python harness.py > current.json
if [ ! -f baseline.json ]; then
  cp current.json baseline.json      # 初回はそのまま基準にする
  echo "baseline を作成しました"; exit 0
fi
python compare.py baseline.json current.json