Antigravity × OpenTelemetry で可観測性を組み立てる — 計装・サンプリング・カーディナリティ・AI異常検知の実運用メモ

# Antigravity のターミナルで実行
# エージェントにプロジェクト構造を解析させる
@agents analyze the project structure and identify all HTTP endpoints, 
database queries, and external API calls that need instrumentation
 
# OpenTelemetry パッケージのインストール
npm install @opentelemetry/sdk-node \
  @opentelemetry/api \
  @opentelemetry/auto-instrumentations-node \
  @opentelemetry/exporter-trace-otlp-http \
  @opentelemetry/exporter-metrics-otlp-http \
  @opentelemetry/sdk-metrics \
  @opentelemetry/resources \
  @opentelemetry/semantic-conventions

// src/instrumentation.ts
// Antigravity エージェントが生成する OTel 初期化コード
 
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
import { Resource } from '@opentelemetry/resources';
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
 
const resource = new Resource({
  [ATTR_SERVICE_NAME]: process.env.OTEL_SERVICE_NAME || 'my-api-service',
  [ATTR_SERVICE_VERSION]: process.env.npm_package_version || '1.0.0',
  'deployment.environment': process.env.NODE_ENV || 'development',
});
 
const traceExporter = new OTLPTraceExporter({
  url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT + '/v1/traces',
});
 
const metricExporter = new OTLPMetricExporter({
  url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT + '/v1/metrics',
});
 
const sdk = new NodeSDK({
  resource,
  traceExporter,
  metricReader: new PeriodicExportingMetricReader({
    exporter: metricExporter,
    exportIntervalMillis: 30000, // 30秒ごとにメトリクスをエクスポート
  }),
  instrumentations: [
    getNodeAutoInstrumentations({
      '@opentelemetry/instrumentation-http': {
        ignoreIncomingPaths: ['/health', '/ready'], // ヘルスチェックは除外
      },
      '@opentelemetry/instrumentation-express': { enabled: true },
      '@opentelemetry/instrumentation-pg': { enabled: true },
      '@opentelemetry/instrumentation-redis': { enabled: true },
    }),
  ],
});
 
sdk.start();
 
// グレースフルシャットダウン
process.on('SIGTERM', () => {
  sdk.shutdown()
    .then(() => console.log('OTel SDK shut down successfully'))
    .catch((err) => console.error('Error shutting down OTel SDK', err))
    .finally(() => process.exit(0));
});
 
export default sdk;

// package.json の scripts を更新
{
  "scripts": {
    "start": "node --require ./src/instrumentation.ts src/index.ts",
    "dev": "tsx --require ./src/instrumentation.ts src/index.ts"
  }
}

// src/services/payment.service.ts
// 決済処理にカスタムスパンとメトリクスを追加
 
import { trace, metrics, SpanStatusCode } from '@opentelemetry/api';
 
const tracer = trace.getTracer('payment-service');
const meter = metrics.getMeter('payment-service');
 
// カスタムメトリクス定義
const paymentCounter = meter.createCounter('payments.processed', {
  description: '処理された決済の総数',
  unit: '1',
});
 
const paymentDuration = meter.createHistogram('payments.duration', {
  description: '決済処理にかかった時間',
  unit: 'ms',
});
 
const activePayments = meter.createUpDownCounter('payments.active', {
  description: '現在処理中の決済数',
});
 
export async function processPayment(
  userId: string,
  amount: number,
  currency: string
): Promise<PaymentResult> {
  // カスタムスパンを開始
  return tracer.startActiveSpan('payment.process', async (span) => {
    const startTime = Date.now();
    activePayments.add(1);
 
    try {
      // スパンに属性を追加（トレースで検索可能になる）
      span.setAttributes({
        'payment.user_id': userId,
        'payment.amount': amount,
        'payment.currency': currency,
      });
 
      // Stripe API 呼び出し（自動計装で HTTP スパンが追加される）
      const result = await stripe.paymentIntents.create({
        amount: Math.round(amount * 100),
        currency,
        metadata: { userId },
      });
 
      // 成功メトリクスを記録
      paymentCounter.add(1, {
        status: 'success',
        currency,
      });
 
      span.setStatus({ code: SpanStatusCode.OK });
      return { success: true, intentId: result.id };
 
    } catch (error) {
      // エラーメトリクスを記録
      paymentCounter.add(1, {
        status: 'error',
        currency,
        error_type: error instanceof Error ? error.name : 'unknown',
      });
 
      span.setStatus({
        code: SpanStatusCode.ERROR,
        message: error instanceof Error ? error.message : 'Payment failed',
      });
      span.recordException(error as Error);
 
      throw error;
    } finally {
      activePayments.add(-1);
      paymentDuration.record(Date.now() - startTime, { currency });
      span.end();
    }
  });
}

// src/lib/logger.ts
// pino ロガーにトレースコンテキストを注入
 
import pino from 'pino';
import { trace, context } from '@opentelemetry/api';
 
function getTraceContext() {
  const activeSpan = trace.getSpan(context.active());
  if (!activeSpan) return {};
 
  const spanContext = activeSpan.spanContext();
  return {
    trace_id: spanContext.traceId,
    span_id: spanContext.spanId,
    trace_flags: spanContext.traceFlags,
  };
}
 
export const logger = pino({
  level: process.env.LOG_LEVEL || 'info',
  mixin() {
    return getTraceContext();
  },
  formatters: {
    level(label) {
      return { severity: label.toUpperCase() };
    },
  },
  transport: process.env.NODE_ENV === 'development'
    ? { target: 'pino-pretty' }
    : undefined,
});
 
// 使用例
// logger.info({ userId, action: 'login' }, 'User logged in');
// → 出力に trace_id と span_id が自動付与される

# docker-compose.otel.yml
# Antigravity エージェントが生成する OTel コレクター + Grafana Stack
 
version: '3.8'
services:
  otel-collector:
    image: otel/opentelemetry-collector-contrib:latest
    command: ['--config', '/etc/otel/config.yaml']
    volumes:
      - ./otel-config.yaml:/etc/otel/config.yaml
    ports:
      - '4318:4318'   # OTLP HTTP receiver
      - '4317:4317'   # OTLP gRPC receiver
      - '8889:8889'   # Prometheus exporter
    depends_on:
      - tempo
      - loki
 
  tempo:
    image: grafana/tempo:latest
    command: ['-config.file=/etc/tempo/config.yaml']
    volumes:
      - ./tempo-config.yaml:/etc/tempo/config.yaml
      - tempo-data:/var/tempo
    ports:
      - '3200:3200'   # Tempo API
 
  prometheus:
    image: prom/prometheus:latest
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    ports:
      - '9090:9090'
 
  loki:
    image: grafana/loki:latest
    ports:
      - '3100:3100'
 
  grafana:
    image: grafana/grafana:latest
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
    ports:
      - '3001:3000'
    volumes:
      - ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
    depends_on:
      - prometheus
      - tempo
      - loki
 
volumes:
  tempo-data:

# otel-config.yaml
# OpenTelemetry Collector の設定
 
receivers:
  otlp:
    protocols:
      http:
        endpoint: 0.0.0.0:4318
      grpc:
        endpoint: 0.0.0.0:4317
 
processors:
  batch:
    timeout: 5s
    send_batch_size: 1024
  memory_limiter:
    check_interval: 1s
    limit_mib: 512
    spike_limit_mib: 128
  # 属性にサービス情報を追加
  attributes:
    actions:
      - key: environment
        value: production
        action: upsert
 
exporters:
  otlp/tempo:
    endpoint: tempo:4317
    tls:
      insecure: true
  prometheusremotewrite:
    endpoint: http://prometheus:9090/api/v1/write
  loki:
    endpoint: http://loki:3100/loki/api/v1/push
 
service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [otlp/tempo]
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [prometheusremotewrite]
    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch, attributes]
      exporters: [loki]

// src/observability/anomaly-detector.ts
// メトリクスの異常を検知するモジュール
 
interface MetricDataPoint {
  timestamp: number;
  value: number;
}
 
interface AnomalyAlert {
  metric: string;
  severity: 'warning' | 'critical';
  currentValue: number;
  threshold: number;
  message: string;
  suggestedAction: string;
}
 
export class AnomalyDetector {
  private readonly windowSize = 60; // 過去60データポイント
  private readonly stdDevMultiplier = 2.5;
 
  /**
   * Z-score ベースの異常検知
   * 過去データの平均・標準偏差から逸脱したポイントを検出する
   */
  detectAnomaly(
    metricName: string,
    dataPoints: MetricDataPoint[]
  ): AnomalyAlert | null {
    if (dataPoints.length < this.windowSize) return null;
 
    const window = dataPoints.slice(-this.windowSize);
    const values = window.map((dp) => dp.value);
 
    const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
    const stdDev = Math.sqrt(
      values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / values.length
    );
 
    const latest = dataPoints[dataPoints.length - 1].value;
    const zScore = stdDev === 0 ? 0 : Math.abs(latest - mean) / stdDev;
 
    if (zScore > this.stdDevMultiplier) {
      const severity = zScore > 4 ? 'critical' : 'warning';
      return {
        metric: metricName,
        severity,
        currentValue: latest,
        threshold: mean + stdDev * this.stdDevMultiplier,
        message: `${metricName} が異常値を検出: ${latest.toFixed(2)} (平均: ${mean.toFixed(2)}, Z-score: ${zScore.toFixed(2)})`,
        suggestedAction: this.getSuggestedAction(metricName, severity),
      };
    }
 
    return null;
  }
 
  private getSuggestedAction(
    metric: string,
    severity: string
  ): string {
    const actions: Record<string, string> = {
      'http.server.duration': 'スロークエリの確認 → コネクションプールの拡張 → キャッシュレイヤーの追加',
      'http.server.request.error_rate': 'エラーログの確認 → 直近のデプロイをロールバック → 依存サービスの状態確認',
      'process.runtime.nodejs.memory.heap_used': 'メモリリークの調査 → ヒープスナップショット取得 → Pod の再起動',
      'payments.duration': 'Stripe API のステータス確認 → リトライ設定の見直し → フォールバック処理の有効化',
    };
    return actions[metric] || '詳細調査が必要です。トレースとログを確認してください。';
  }
}

// src/observability/auto-remediation.ts
// 異常検知をトリガーに自動修復アクションを実行
 
import { logger } from '../lib/logger';
import { AnomalyAlert } from './anomaly-detector';
 
interface RemediationAction {
  name: string;
  condition: (alert: AnomalyAlert) => boolean;
  execute: (alert: AnomalyAlert) => Promise<RemediationResult>;
}
 
interface RemediationResult {
  success: boolean;
  action: string;
  details: string;
}
 
export class AutoRemediation {
  private actions: RemediationAction[] = [];
  private executionLog: RemediationResult[] = [];
 
  registerAction(action: RemediationAction): void {
    this.actions.push(action);
    logger.info({ actionName: action.name }, 'Remediation action registered');
  }
 
  async handleAlert(alert: AnomalyAlert): Promise<RemediationResult[]> {
    const results: RemediationResult[] = [];
 
    for (const action of this.actions) {
      if (action.condition(alert)) {
        logger.info(
          { alert: alert.metric, action: action.name },
          'Executing remediation action'
        );
 
        try {
          const result = await action.execute(alert);
          results.push(result);
          this.executionLog.push(result);
 
          if (result.success) {
            logger.info(
              { action: action.name, details: result.details },
              'Remediation succeeded'
            );
            break; // 成功したら後続アクションはスキップ
          }
        } catch (error) {
          const failResult: RemediationResult = {
            success: false,
            action: action.name,
            details: error instanceof Error ? error.message : 'Unknown error',
          };
          results.push(failResult);
          logger.error(
            { action: action.name, error },
            'Remediation action failed'
          );
        }
      }
    }
 
    return results;
  }
}
 
// 使用例: 自動修復アクションの登録
const remediation = new AutoRemediation();
 
// メモリ使用量が閾値を超えた場合、キャッシュをクリア
remediation.registerAction({
  name: 'clear-application-cache',
  condition: (alert) =>
    alert.metric.includes('memory') && alert.severity === 'warning',
  execute: async (alert) => {
    // アプリケーションキャッシュのクリアロジック
    globalCache.clear();
    return {
      success: true,
      action: 'clear-application-cache',
      details: `Cache cleared. Memory was at ${alert.currentValue}MB`,
    };
  },
});
 
// エラーレートが critical の場合、サーキットブレーカーを発動
remediation.registerAction({
  name: 'activate-circuit-breaker',
  condition: (alert) =>
    alert.metric.includes('error_rate') && alert.severity === 'critical',
  execute: async (alert) => {
    circuitBreaker.open();
    return {
      success: true,
      action: 'activate-circuit-breaker',
      details: `Circuit breaker activated. Error rate: ${alert.currentValue}%`,
    };
  },
});

// scripts/generate-grafana-dashboard.ts
// Antigravity エージェントが生成するダッシュボード設定スクリプト
 
interface GrafanaPanel {
  title: string;
  type: 'timeseries' | 'stat' | 'gauge' | 'table';
  targets: { expr: string; legendFormat: string }[];
  gridPos: { h: number; w: number; x: number; y: number };
}
 
function generateDashboard(serviceName: string): object {
  const panels: GrafanaPanel[] = [
    {
      title: 'Request Rate (per second)',
      type: 'timeseries',
      targets: [{
        expr: `rate(http_server_request_duration_seconds_count{service_name="${serviceName}"}[5m])`,
        legendFormat: '{{http_method}} {{http_route}}',
      }],
      gridPos: { h: 8, w: 12, x: 0, y: 0 },
    },
    {
      title: 'P95 Latency',
      type: 'timeseries',
      targets: [{
        expr: `histogram_quantile(0.95, rate(http_server_request_duration_seconds_bucket{service_name="${serviceName}"}[5m]))`,
        legendFormat: '{{http_route}}',
      }],
      gridPos: { h: 8, w: 12, x: 12, y: 0 },
    },
    {
      title: 'Error Rate',
      type: 'stat',
      targets: [{
        expr: `sum(rate(http_server_request_duration_seconds_count{service_name="${serviceName}",http_status_code=~"5.."}[5m])) / sum(rate(http_server_request_duration_seconds_count{service_name="${serviceName}"}[5m])) * 100`,
        legendFormat: 'Error %',
      }],
      gridPos: { h: 4, w: 6, x: 0, y: 8 },
    },
    {
      title: 'Active Payments',
      type: 'gauge',
      targets: [{
        expr: `payments_active{service_name="${serviceName}"}`,
        legendFormat: 'Active',
      }],
      gridPos: { h: 4, w: 6, x: 6, y: 8 },
    },
  ];
 
  return {
    dashboard: {
      title: `${serviceName} — Observability Dashboard`,
      tags: ['opentelemetry', 'auto-generated'],
      panels,
      time: { from: 'now-1h', to: 'now' },
      refresh: '10s',
    },
  };
}
 
// 実行: npx tsx scripts/generate-grafana-dashboard.ts > dashboard.json
const dashboard = generateDashboard('my-api-service');
console.log(JSON.stringify(dashboard, null, 2));

# otel-collector-config.yaml
processors:
  tail_sampling:
    decision_wait: 10s
    num_traces: 50000
    expected_new_traces_per_sec: 1000
    policies:
      # 1. エラーを含むトレースは必ず残す
      - name: errors-always
        type: status_code
        status_code: { status_codes: [ERROR] }
      # 2. 遅いトレースは必ず残す
      - name: slow-traces
        type: latency
        latency: { threshold_ms: 500 }
      # 3. それ以外は 5% だけ残す（ベースライン把握用）
      - name: baseline
        type: probabilistic
        probabilistic: { sampling_percentage: 5 }

// ❌ 危険: userId と path が高カーディナリティ
requestCounter.add(1, {
  'http.route': req.path,        // /users/8a3f-… が全部別ラベル
  'user.id': req.user.id,        // ユーザー数だけ時系列が増える
});

// ✅ メトリクスは低カーディナリティに保つ
requestCounter.add(1, {
  'http.route': req.route?.path ?? 'unknown',  // /users/:id
  'http.status_code': res.statusCode,
  'service.version': process.env.APP_VERSION ?? 'dev',
});
 
// 個体識別はスパン側へ
const span = trace.getActiveSpan();
span?.setAttribute('user.id', req.user.id);
span?.setAttribute('request.id', req.headers['x-request-id'] as string);

processors:
  # 想定外の高カーディナリティ属性を落とす
  attributes/scrub:
    actions:
      - key: user.id
        action: delete
      - key: session.id
        action: delete
  # 時系列数の上限を設けて、超えたら捨てる
  memory_limiter:
    check_interval: 1s
    limit_mib: 512
    spike_limit_mib: 128