When the Tech-Debt Score Drops but the Same Files Keep Breaking — Field Notes on Instrumenting Fan-in and Churn

// scripts/hotspot-score.ts
// Surface risk hotspots via fan-in (inbound refs) x churn (change frequency).
// The point is to catch the "simple but frequently broken" files that
// static complexity alone lets sink.
 
import madge from "madge";
import { execSync } from "child_process";
import { writeFileSync, mkdirSync } from "fs";
 
interface Hotspot {
  file: string;
  fanIn: number;        // number of modules referencing this file
  churn: number;        // commits in the last 90 days that touched it
  linesChanged: number; // added + removed lines over the same window
  hotspotScore: number; // log-compressed fanIn x churn
}
 
// Count, per file, how many commits it appeared in over the window
function collectChurn(sinceDays = 90): Map<string, { commits: number; lines: number }> {
  const since = `--since=${sinceDays}.days.ago`;
  // --numstat emits "added\tremoved\tpath"; binaries show "-", so skip them.
  const raw = execSync(`git log ${since} --numstat --pretty=format:__COMMIT__`, {
    encoding: "utf8",
    maxBuffer: 64 * 1024 * 1024,
  });
 
  const churn = new Map<string, { commits: number; lines: number }>();
  const seenInCommit = new Set<string>();
 
  for (const line of raw.split("\n")) {
    if (line === "__COMMIT__") {
      seenInCommit.clear();
      continue;
    }
    const m = line.match(/^(\d+|-)\t(\d+|-)\t(.+)$/);
    if (!m) continue;
    const added = m[1] === "-" ? 0 : parseInt(m[1], 10);
    const removed = m[2] === "-" ? 0 : parseInt(m[2], 10);
    const path = m[3];
    if (!/\.(ts|tsx)$/.test(path)) continue;
 
    const cur = churn.get(path) ?? { commits: 0, lines: 0 };
    // avoid double-counting a file within the same commit
    if (!seenInCommit.has(path)) {
      cur.commits += 1;
      seenInCommit.add(path);
    }
    cur.lines += added + removed;
    churn.set(path, cur);
  }
  return churn;
}
 
export async function computeHotspots(srcPath = "./src"): Promise<Hotspot[]> {
  const result = await madge(srcPath, {
    fileExtensions: ["ts", "tsx"],
    excludeRegExp: [/\.test\./, /\.spec\./, /__tests__/],
    tsConfig: "./tsconfig.json",
  });
  const graph = result.obj();
 
  // Aggregate fan-in (how often each dependency is referenced)
  const fanIn = new Map<string, number>();
  for (const deps of Object.values(graph)) {
    for (const dep of deps) fanIn.set(dep, (fanIn.get(dep) ?? 0) + 1);
  }
 
  const churn = collectChurn(90);
 
  const hotspots: Hotspot[] = [];
  for (const [file, fi] of fanIn.entries()) {
    const c = churn.get(file);
    if (!c) continue; // untouched for 90 days -> out of scope this round
    // both fan-in and churn are long-tailed; compress with log before multiplying
    const score = Math.log2(fi + 1) * Math.log2(c.commits + 1);
    hotspots.push({
      file,
      fanIn: fi,
      churn: c.commits,
      linesChanged: c.lines,
      hotspotScore: Number(score.toFixed(2)),
    });
  }
 
  hotspots.sort((a, b) => b.hotspotScore - a.hotspotScore);
  mkdirSync(".architecture-analysis/scores", { recursive: true });
  writeFileSync(
    ".architecture-analysis/scores/hotspots.json",
    JSON.stringify(hotspots.slice(0, 50), null, 2)
  );
  return hotspots;
}
 
computeHotspots().then((h) => {
  console.log("Top hotspots:");
  for (const s of h.slice(0, 10)) {
    console.log(`  ${s.file}  fanIn=${s.fanIn} churn=${s.churn} score=${s.hotspotScore}`);
  }
});

// scripts/reconcile-score-vs-incidents.ts
// Measure how well the top-N of a ranking captures the files that
// incidents actually patched. Compares the static-complexity ranking
// and the hotspot ranking on the same footing.
 
import { execSync } from "child_process";
import { readFileSync } from "fs";
 
// Extract the set of files that incident-response commits touched
function incidentFiles(sinceDays = 180): Set<string> {
  const raw = execSync(
    `git log --since=${sinceDays}.days.ago --grep='incident/' -i --name-only --pretty=format:__C__`,
    { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }
  );
  const files = new Set<string>();
  for (const line of raw.split("\n")) {
    if (line === "__C__" || line.trim() === "") continue;
    if (/\.(ts|tsx)$/.test(line)) files.add(line);
  }
  return files;
}
 
// Recall@N: of the incident files, how many appear in the ranking's top N
function recallAtN(ranking: string[], truth: Set<string>, n: number): number {
  const topN = new Set(ranking.slice(0, n));
  let hit = 0;
  for (const f of truth) if (topN.has(f)) hit++;
  return truth.size === 0 ? 0 : Number((hit / truth.size).toFixed(3));
}
 
const truth = incidentFiles(180);
 
const hotspots: { file: string }[] = JSON.parse(
  readFileSync(".architecture-analysis/scores/hotspots.json", "utf8")
);
const complexity: { filePath: string }[] = JSON.parse(
  readFileSync(".architecture-analysis/scores/debt-scores.json", "utf8")
);
 
const hotspotRanking = hotspots.map((h) => h.file);
const complexityRanking = complexity.map((c) => c.filePath);
 
for (const n of [10, 20, 30]) {
  console.log(
    `Recall@${n}  complexity=${recallAtN(complexityRanking, truth, n)}  ` +
      `hotspot=${recallAtN(hotspotRanking, truth, n)}`
  );
}

# Architecture Analyst Agent (excerpt, revised)
 
## Prioritization Rules
- Primary key for refactoring order is hotspotScore in hotspots.json.
- Complexity (debt-scores.json) is a tiebreaker only; never orders alone.
- Before proposing, read the reconcile report; if hotspot Recall@20 is
  below complexity's, propose fixing the measurement logic before code.
- Attach "fanIn / churn / recent-incident?" as evidence for each item.
 
## Output
- Repay roadmap -> .architecture-analysis/reports/refactor-roadmap.md
- Each item includes an effort estimate and the dependency-ripple file list