Antigravity × Multi-Provider LLM Failover — A Production Guide for Gemini, Claude, and Local Gemma

// src/llm/provider.ts
// Purpose: give every LLM provider a shared contract the router can treat uniformly
 
export interface LLMRequest {
  messages: Array<{ role: "system" | "user" | "assistant"; content: string }>;
  maxTokens?: number;
  temperature?: number;
  jsonSchema?: Record<string, unknown>;  // capability axis: needs structured output?
  minContextTokens?: number;              // capability axis: required input window
}
 
export interface LLMResponse {
  content: string;
  inputTokens: number;
  outputTokens: number;
  provider: string;       // "gemini" | "claude" | "gemma-local"
  latencyMs: number;
  attemptCount: number;   // which attempt succeeded (for audit)
}
 
export interface LLMProvider {
  readonly name: string;
  readonly capabilities: {
    maxContext: number;
    supportsJsonSchema: boolean;
    requiresNetwork: boolean;
  };
  readonly cost: {
    inputPer1M: number;   // USD
    outputPer1M: number;  // USD
  };
  invoke(req: LLMRequest, signal: AbortSignal): Promise<LLMResponse>;
  healthCheck(): Promise<boolean>;
}

// src/llm/gemini-provider.ts
import { GoogleGenerativeAI } from "@google/generative-ai";
import type { LLMProvider, LLMRequest, LLMResponse } from "./provider";
 
export class GeminiProvider implements LLMProvider {
  readonly name = "gemini";
  readonly capabilities = {
    maxContext: 1_048_576,
    supportsJsonSchema: true,
    requiresNetwork: true,
  };
  readonly cost = { inputPer1M: 0.15, outputPer1M: 0.6 };
 
  private client: GoogleGenerativeAI;
  private modelName = "gemini-2.5-flash";
 
  constructor(apiKey: string) {
    this.client = new GoogleGenerativeAI(apiKey);
  }
 
  async invoke(req: LLMRequest, signal: AbortSignal): Promise<LLMResponse> {
    const start = Date.now();
    const model = this.client.getGenerativeModel({
      model: this.modelName,
      generationConfig: {
        maxOutputTokens: req.maxTokens ?? 4096,
        temperature: req.temperature ?? 0.7,
        ...(req.jsonSchema ? { responseMimeType: "application/json", responseSchema: req.jsonSchema } : {}),
      },
    });
 
    // Pass AbortSignal into the SDK so the underlying socket closes on timeout
    const result = await model.generateContent({
      contents: req.messages.map((m) => ({ role: m.role === "assistant" ? "model" : m.role, parts: [{ text: m.content }] })),
    }, { signal });
 
    const text = result.response.text();
    const usage = result.response.usageMetadata;
 
    return {
      content: text,
      inputTokens: usage?.promptTokenCount ?? 0,
      outputTokens: usage?.candidatesTokenCount ?? 0,
      provider: this.name,
      latencyMs: Date.now() - start,
      attemptCount: 1,
    };
  }
 
  async healthCheck(): Promise<boolean> {
    try {
      const controller = new AbortController();
      const timer = setTimeout(() => controller.abort(), 3000);
      await this.invoke({ messages: [{ role: "user", content: "ok" }], maxTokens: 5 }, controller.signal);
      clearTimeout(timer);
      return true;
    } catch {
      return false;
    }
  }
}

// src/llm/circuit-breaker.ts
// Purpose: track consecutive failures and skip a provider during cooldown
 
export type CircuitState = "closed" | "open" | "half-open";
 
export interface CircuitBreakerConfig {
  failureThreshold: number;     // fail N times in a row to trip
  openDurationMs: number;       // how long to stay open before testing
  halfOpenSuccessCount: number; // successes required in half-open to close
}
 
export class CircuitBreaker {
  private state: CircuitState = "closed";
  private consecutiveFailures = 0;
  private consecutiveHalfOpenSuccess = 0;
  private openedAt = 0;
 
  constructor(private readonly name: string, private readonly config: CircuitBreakerConfig) {}
 
  canExecute(): boolean {
    if (this.state === "closed") return true;
    if (this.state === "open") {
      if (Date.now() - this.openedAt >= this.config.openDurationMs) {
        this.state = "half-open";
        this.consecutiveHalfOpenSuccess = 0;
        return true;
      }
      return false;
    }
    return true;  // half-open: allow a limited trial
  }
 
  recordSuccess(): void {
    if (this.state === "half-open") {
      this.consecutiveHalfOpenSuccess++;
      if (this.consecutiveHalfOpenSuccess >= this.config.halfOpenSuccessCount) {
        this.reset();
      }
      return;
    }
    this.consecutiveFailures = 0;
  }
 
  recordFailure(): void {
    if (this.state === "half-open") {
      // any failure in half-open pops us straight back to open
      this.state = "open";
      this.openedAt = Date.now();
      return;
    }
    this.consecutiveFailures++;
    if (this.consecutiveFailures >= this.config.failureThreshold) {
      this.state = "open";
      this.openedAt = Date.now();
    }
  }
 
  private reset(): void {
    this.state = "closed";
    this.consecutiveFailures = 0;
    this.consecutiveHalfOpenSuccess = 0;
  }
 
  getState(): CircuitState {
    return this.state;
  }
}

// src/llm/router.ts
// Purpose: rank providers by capability → latency → cost, and fail over with breakers
 
import type { LLMProvider, LLMRequest, LLMResponse } from "./provider";
import { CircuitBreaker } from "./circuit-breaker";
 
interface ProviderSlot {
  provider: LLMProvider;
  breaker: CircuitBreaker;
  recentLatencyMs: number[];
}
 
export class LLMRouter {
  private slots: ProviderSlot[] = [];
  private readonly latencyWindow = 20;
 
  constructor(providers: LLMProvider[]) {
    this.slots = providers.map((p) => ({
      provider: p,
      breaker: new CircuitBreaker(p.name, {
        failureThreshold: 3,
        openDurationMs: 30000,
        halfOpenSuccessCount: 2,
      }),
      recentLatencyMs: [],
    }));
  }
 
  async execute(req: LLMRequest, timeoutMs = 20000): Promise<LLMResponse> {
    const candidates = this.rankProviders(req);
    if (candidates.length === 0) {
      throw new Error("No capable provider available for this request");
    }
 
    const errors: Array<{ provider: string; error: unknown }> = [];
    let attemptCount = 0;
 
    for (const slot of candidates) {
      if (!slot.breaker.canExecute()) continue;  // skip open circuits
      attemptCount++;
      const controller = new AbortController();
      const timer = setTimeout(() => controller.abort(), timeoutMs);
      try {
        const res = await slot.provider.invoke(req, controller.signal);
        clearTimeout(timer);
        slot.breaker.recordSuccess();
        this.recordLatency(slot, res.latencyMs);
        return { ...res, attemptCount };
      } catch (err) {
        clearTimeout(timer);
        slot.breaker.recordFailure();
        errors.push({ provider: slot.provider.name, error: err });
      }
    }
 
    const detail = errors.map((e) => `${e.provider}: ${String(e.error)}`).join(" / ");
    throw new Error(`All providers failed after ${attemptCount} attempts — ${detail}`);
  }
 
  private rankProviders(req: LLMRequest): ProviderSlot[] {
    // Axis 1: capability filter
    const capable = this.slots.filter((s) => {
      if (req.jsonSchema && !s.provider.capabilities.supportsJsonSchema) return false;
      if (req.minContextTokens && req.minContextTokens > s.provider.capabilities.maxContext) return false;
      return true;
    });
 
    // Axis 2 + 3: latency median, break ties by output cost
    return capable.sort((a, b) => {
      const diff = this.medianLatency(a) - this.medianLatency(b);
      if (Math.abs(diff) > 500) return diff;  // >500ms: latency decides
      return a.provider.cost.outputPer1M - b.provider.cost.outputPer1M;
    });
  }
 
  private medianLatency(slot: ProviderSlot): number {
    if (slot.recentLatencyMs.length === 0) return 1500;
    const sorted = [...slot.recentLatencyMs].sort((a, b) => a - b);
    return sorted[Math.floor(sorted.length / 2)];
  }
 
  private recordLatency(slot: ProviderSlot, latencyMs: number): void {
    slot.recentLatencyMs.push(latencyMs);
    if (slot.recentLatencyMs.length > this.latencyWindow) slot.recentLatencyMs.shift();
  }
}

// src/main.ts
import { GeminiProvider } from "./llm/gemini-provider";
import { ClaudeProvider } from "./llm/claude-provider";
import { GemmaLocalProvider } from "./llm/gemma-local-provider";
import { LLMRouter } from "./llm/router";
 
const router = new LLMRouter([
  new GeminiProvider(process.env.GEMINI_API_KEY!),
  new ClaudeProvider(process.env.ANTHROPIC_API_KEY!),
  new GemmaLocalProvider("http://localhost:1234/v1"),  // LM Studio
]);
 
const res = await router.execute({
  messages: [{ role: "user", content: "Generate a product description as JSON" }],
  jsonSchema: { type: "object", properties: { name: { type: "string" }, description: { type: "string" } } },
  maxTokens: 500,
});
 
console.log(`Provider: ${res.provider}, Attempts: ${res.attemptCount}, Latency: ${res.latencyMs}ms`);
// e.g. "Provider: claude, Attempts: 2, Latency: 1843ms"
// One line tells you Gemini failed and Claude picked up.

## Incident note YYYY-MM-DD
 
### What happened (facts only)
- Start / recovery time:
- Affected provider(s):
- Peak fallback rate:
- User-visible error count:
 
### How detection worked
- First signal (alert / support ticket / my own eyes):
- Did the alert fire as expected:
 
### How the router behaved
- Did the circuit open / after how many seconds:
- Was the fallback target appropriate:
- Any unexpected behavior:
 
### One next step (only one)
- Exactly one of: config change / code fix / new monitoring: