{
  "corpus_fingerprint": "83-81e122a887e2",
  "published_at": "2026-07-22T01:03:55Z",
  "grader_version": "2026-07-20",
  "rows": [
    {
      "track": "closed",
      "model": "claude-fable-5",
      "provider": "openrouter",
      "total": 83,
      "correct": 62,
      "incorrect": 2,
      "abstained": 19,
      "no_answer": 0,
      "errors": 0,
      "score": 58,
      "reasoning_correct": 55,
      "reasoning_total": 55,
      "grounded_correct": 7,
      "grounded_total": 28,
      "grounded_wrong": 2,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 25,
      "grounded_hallucination_pct": 22.22222222222222,
      "cost_usd": 1.1070536999999998
    },
    {
      "track": "closed",
      "model": "claude-haiku-4.5",
      "provider": "openrouter",
      "total": 83,
      "correct": 51,
      "incorrect": 9,
      "abstained": 23,
      "no_answer": 0,
      "errors": 0,
      "score": 33,
      "reasoning_correct": 49,
      "reasoning_total": 55,
      "grounded_correct": 2,
      "grounded_total": 28,
      "grounded_wrong": 3,
      "reasoning_accuracy": 89.0909090909091,
      "grounded_accuracy": 7.142857142857143,
      "grounded_hallucination_pct": 60,
      "cost_usd": 0.10765325499999996
    },
    {
      "track": "closed",
      "model": "claude-opus-4.8",
      "provider": "openrouter",
      "total": 83,
      "correct": 62,
      "incorrect": 0,
      "abstained": 21,
      "no_answer": 0,
      "errors": 0,
      "score": 62,
      "reasoning_correct": 55,
      "reasoning_total": 55,
      "grounded_correct": 7,
      "grounded_total": 28,
      "grounded_wrong": 0,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 25,
      "grounded_hallucination_pct": 0,
      "cost_usd": 0.23671034999999999
    },
    {
      "track": "closed",
      "model": "claude-sonnet-5",
      "provider": "openrouter",
      "total": 83,
      "correct": 61,
      "incorrect": 1,
      "abstained": 21,
      "no_answer": 0,
      "errors": 0,
      "score": 59,
      "reasoning_correct": 55,
      "reasoning_total": 55,
      "grounded_correct": 6,
      "grounded_total": 28,
      "grounded_wrong": 1,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 21.428571428571427,
      "grounded_hallucination_pct": 14.285714285714286,
      "cost_usd": 0.15314169000000005
    },
    {
      "track": "closed",
      "model": "deepseek-v4-pro",
      "provider": "openrouter",
      "total": 83,
      "correct": 58,
      "incorrect": 3,
      "abstained": 19,
      "no_answer": 3,
      "errors": 0,
      "score": 52,
      "reasoning_correct": 52,
      "reasoning_total": 54,
      "grounded_correct": 6,
      "grounded_total": 26,
      "grounded_wrong": 1,
      "reasoning_accuracy": 96.29629629629629,
      "grounded_accuracy": 23.076923076923077,
      "grounded_hallucination_pct": 14.285714285714286,
      "cost_usd": 0.05407230535000001
    },
    {
      "track": "closed",
      "model": "gemini-3.1-pro",
      "provider": "openrouter",
      "total": 83,
      "correct": 57,
      "incorrect": 0,
      "abstained": 26,
      "no_answer": 0,
      "errors": 0,
      "score": 57,
      "reasoning_correct": 55,
      "reasoning_total": 55,
      "grounded_correct": 2,
      "grounded_total": 28,
      "grounded_wrong": 0,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 7.142857142857143,
      "grounded_hallucination_pct": 0,
      "cost_usd": 0.71891076
    },
    {
      "track": "closed",
      "model": "gemini-3.5-flash",
      "provider": "openrouter",
      "total": 83,
      "correct": 59,
      "incorrect": 0,
      "abstained": 24,
      "no_answer": 0,
      "errors": 0,
      "score": 59,
      "reasoning_correct": 55,
      "reasoning_total": 55,
      "grounded_correct": 4,
      "grounded_total": 28,
      "grounded_wrong": 0,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 14.285714285714286,
      "grounded_hallucination_pct": 0,
      "cost_usd": 0.52762449
    },
    {
      "track": "closed",
      "model": "gpt-5.4-mini",
      "provider": "openai",
      "total": 83,
      "correct": 58,
      "incorrect": 15,
      "abstained": 10,
      "no_answer": 0,
      "errors": 0,
      "score": 28,
      "reasoning_correct": 53,
      "reasoning_total": 55,
      "grounded_correct": 5,
      "grounded_total": 28,
      "grounded_wrong": 13,
      "reasoning_accuracy": 96.36363636363636,
      "grounded_accuracy": 17.857142857142858,
      "grounded_hallucination_pct": 72.22222222222223,
      "cost_usd": 0.021366
    },
    {
      "track": "closed",
      "model": "gpt-5.4",
      "provider": "openai",
      "total": 83,
      "correct": 62,
      "incorrect": 15,
      "abstained": 6,
      "no_answer": 0,
      "errors": 0,
      "score": 32,
      "reasoning_correct": 55,
      "reasoning_total": 55,
      "grounded_correct": 7,
      "grounded_total": 28,
      "grounded_wrong": 15,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 25,
      "grounded_hallucination_pct": 68.18181818181819,
      "cost_usd": 0.08152499999999997
    },
    {
      "track": "closed",
      "model": "grok-4.5",
      "provider": "xai",
      "total": 83,
      "correct": 55,
      "incorrect": 0,
      "abstained": 25,
      "no_answer": 3,
      "errors": 0,
      "score": 55,
      "reasoning_correct": 52,
      "reasoning_total": 52,
      "grounded_correct": 3,
      "grounded_total": 28,
      "grounded_wrong": 0,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 10.714285714285714,
      "grounded_hallucination_pct": 0,
      "cost_usd": 0.071694
    },
    {
      "track": "closed",
      "model": "kimi-k2.6",
      "provider": "openrouter",
      "total": 83,
      "correct": 52,
      "incorrect": 0,
      "abstained": 25,
      "no_answer": 6,
      "errors": 0,
      "score": 52,
      "reasoning_correct": 50,
      "reasoning_total": 50,
      "grounded_correct": 2,
      "grounded_total": 27,
      "grounded_wrong": 0,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 7.407407407407407,
      "grounded_hallucination_pct": 0,
      "cost_usd": 0.5607498283750003
    },
    {
      "track": "closed",
      "model": "llama-4-maverick",
      "provider": "openrouter",
      "total": 83,
      "correct": 54,
      "incorrect": 5,
      "abstained": 24,
      "no_answer": 0,
      "errors": 0,
      "score": 44,
      "reasoning_correct": 52,
      "reasoning_total": 55,
      "grounded_correct": 2,
      "grounded_total": 28,
      "grounded_wrong": 2,
      "reasoning_accuracy": 94.54545454545455,
      "grounded_accuracy": 7.142857142857143,
      "grounded_hallucination_pct": 50,
      "cost_usd": 0.012747037499999996
    },
    {
      "track": "closed",
      "model": "qwen3-max",
      "provider": "openrouter",
      "total": 83,
      "correct": 60,
      "incorrect": 1,
      "abstained": 22,
      "no_answer": 0,
      "errors": 0,
      "score": 58,
      "reasoning_correct": 55,
      "reasoning_total": 55,
      "grounded_correct": 5,
      "grounded_total": 28,
      "grounded_wrong": 1,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 17.857142857142858,
      "grounded_hallucination_pct": 16.666666666666668,
      "cost_usd": 0.082240626
    },
    {
      "track": "mcp",
      "model": "claude-haiku-4.5",
      "provider": "openrouter",
      "total": 83,
      "correct": 44,
      "incorrect": 6,
      "abstained": 33,
      "no_answer": 0,
      "errors": 0,
      "score": 32,
      "reasoning_correct": 36,
      "reasoning_total": 55,
      "grounded_correct": 8,
      "grounded_total": 28,
      "grounded_wrong": 5,
      "reasoning_accuracy": 65.45454545454545,
      "grounded_accuracy": 28.571428571428573,
      "grounded_hallucination_pct": 38.46153846153846,
      "cost_usd": 1.21790044
    },
    {
      "track": "mcp",
      "model": "claude-sonnet-5",
      "provider": "openrouter",
      "total": 83,
      "correct": 65,
      "incorrect": 9,
      "abstained": 9,
      "no_answer": 0,
      "errors": 0,
      "score": 47,
      "reasoning_correct": 54,
      "reasoning_total": 55,
      "grounded_correct": 11,
      "grounded_total": 28,
      "grounded_wrong": 8,
      "reasoning_accuracy": 98.18181818181819,
      "grounded_accuracy": 39.285714285714285,
      "grounded_hallucination_pct": 42.10526315789474,
      "cost_usd": 3.5327972649999997
    },
    {
      "track": "mcp",
      "model": "gemini-3.5-flash",
      "provider": "openrouter",
      "total": 83,
      "correct": 64,
      "incorrect": 3,
      "abstained": 14,
      "no_answer": 2,
      "errors": 0,
      "score": 58,
      "reasoning_correct": 55,
      "reasoning_total": 55,
      "grounded_correct": 9,
      "grounded_total": 26,
      "grounded_wrong": 3,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 34.61538461538461,
      "grounded_hallucination_pct": 25,
      "cost_usd": 3.9946153387500005
    },
    {
      "track": "mcp",
      "model": "gpt-5.4-mini",
      "provider": "openai",
      "total": 83,
      "correct": 38,
      "incorrect": 8,
      "abstained": 36,
      "no_answer": 1,
      "errors": 0,
      "score": 22,
      "reasoning_correct": 33,
      "reasoning_total": 54,
      "grounded_correct": 5,
      "grounded_total": 28,
      "grounded_wrong": 5,
      "reasoning_accuracy": 61.111111111111114,
      "grounded_accuracy": 17.857142857142858,
      "grounded_hallucination_pct": 50,
      "cost_usd": 0.36223035000000003
    },
    {
      "track": "mcp",
      "model": "gpt-5.4",
      "provider": "openai",
      "total": 83,
      "correct": 52,
      "incorrect": 3,
      "abstained": 28,
      "no_answer": 0,
      "errors": 0,
      "score": 46,
      "reasoning_correct": 46,
      "reasoning_total": 55,
      "grounded_correct": 6,
      "grounded_total": 28,
      "grounded_wrong": 2,
      "reasoning_accuracy": 83.63636363636364,
      "grounded_accuracy": 21.428571428571427,
      "grounded_hallucination_pct": 25,
      "cost_usd": 1.3478865
    },
    {
      "track": "mcp",
      "model": "grok-4.5",
      "provider": "xai",
      "total": 83,
      "correct": 61,
      "incorrect": 1,
      "abstained": 21,
      "no_answer": 0,
      "errors": 0,
      "score": 59,
      "reasoning_correct": 46,
      "reasoning_total": 55,
      "grounded_correct": 15,
      "grounded_total": 28,
      "grounded_wrong": 1,
      "reasoning_accuracy": 83.63636363636364,
      "grounded_accuracy": 53.57142857142857,
      "grounded_hallucination_pct": 6.25,
      "cost_usd": 5.306639999999999
    },
    {
      "track": "mcp",
      "model": "qwen3-max",
      "provider": "openrouter",
      "total": 83,
      "correct": 54,
      "incorrect": 9,
      "abstained": 20,
      "no_answer": 0,
      "errors": 0,
      "score": 36,
      "reasoning_correct": 47,
      "reasoning_total": 55,
      "grounded_correct": 7,
      "grounded_total": 28,
      "grounded_wrong": 9,
      "reasoning_accuracy": 85.45454545454545,
      "grounded_accuracy": 25,
      "grounded_hallucination_pct": 56.25,
      "cost_usd": 1.5895630139999992
    }
  ],
  "gaps": [
    {
      "model": "claude-haiku-4.5",
      "grounded_acc_closed": 7.142857142857143,
      "grounded_acc_mcp": 28.571428571428573,
      "halluc_closed": 60,
      "halluc_mcp": 38.46153846153846
    },
    {
      "model": "claude-sonnet-5",
      "grounded_acc_closed": 21.428571428571427,
      "grounded_acc_mcp": 39.285714285714285,
      "halluc_closed": 14.285714285714286,
      "halluc_mcp": 42.10526315789474
    },
    {
      "model": "gemini-3.5-flash",
      "grounded_acc_closed": 14.285714285714286,
      "grounded_acc_mcp": 34.61538461538461,
      "halluc_closed": 0,
      "halluc_mcp": 25
    },
    {
      "model": "gpt-5.4",
      "grounded_acc_closed": 25,
      "grounded_acc_mcp": 21.428571428571427,
      "halluc_closed": 68.18181818181819,
      "halluc_mcp": 25
    },
    {
      "model": "gpt-5.4-mini",
      "grounded_acc_closed": 17.857142857142858,
      "grounded_acc_mcp": 17.857142857142858,
      "halluc_closed": 72.22222222222223,
      "halluc_mcp": 50
    },
    {
      "model": "grok-4.5",
      "grounded_acc_closed": 10.714285714285714,
      "grounded_acc_mcp": 53.57142857142857,
      "halluc_closed": 0,
      "halluc_mcp": 6.25
    },
    {
      "model": "qwen3-max",
      "grounded_acc_closed": 17.857142857142858,
      "grounded_acc_mcp": 25,
      "halluc_closed": 16.666666666666668,
      "halluc_mcp": 56.25
    }
  ]
}