{
  "corpus_fingerprint": "83-81e122a887e2",
  "revision": "2026-07-24",
  "published_at": "2026-07-25T02:52:56Z",
  "grader_version": "2026-07-20",
  "rows": [
    {
      "track": "closed",
      "model": "claude-fable-5",
      "provider": "openrouter",
      "total": 249,
      "correct": 187,
      "incorrect": 4,
      "abstained": 58,
      "no_answer": 0,
      "errors": 0,
      "score": 179,
      "score_per_100": 71.88755020080322,
      "runs": 3,
      "score_per_100_min": 69.87951807228916,
      "score_per_100_max": 73.49397590361446,
      "reasoning_correct": 165,
      "reasoning_total": 165,
      "grounded_correct": 22,
      "grounded_total": 84,
      "grounded_wrong": 4,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 26.19047619047619,
      "grounded_hallucination_pct": 15.384615384615385,
      "cost_usd": 3.2877703499999993
    },
    {
      "track": "closed",
      "model": "claude-haiku-4.5",
      "provider": "openrouter",
      "total": 249,
      "correct": 157,
      "incorrect": 22,
      "abstained": 70,
      "no_answer": 0,
      "errors": 0,
      "score": 113,
      "score_per_100": 45.381526104417674,
      "runs": 3,
      "score_per_100_min": 39.75903614457831,
      "score_per_100_max": 50.602409638554214,
      "reasoning_correct": 150,
      "reasoning_total": 165,
      "grounded_correct": 7,
      "grounded_total": 84,
      "grounded_wrong": 7,
      "reasoning_accuracy": 90.9090909090909,
      "grounded_accuracy": 8.333333333333334,
      "grounded_hallucination_pct": 50,
      "cost_usd": 0.3237246399999999
    },
    {
      "track": "closed",
      "model": "claude-opus-4.8",
      "provider": "openrouter",
      "total": 249,
      "correct": 183,
      "incorrect": 1,
      "abstained": 65,
      "no_answer": 0,
      "errors": 0,
      "score": 181,
      "score_per_100": 72.69076305220884,
      "runs": 3,
      "score_per_100_min": 69.87951807228916,
      "score_per_100_max": 74.6987951807229,
      "reasoning_correct": 164,
      "reasoning_total": 165,
      "grounded_correct": 19,
      "grounded_total": 84,
      "grounded_wrong": 0,
      "reasoning_accuracy": 99.39393939393939,
      "grounded_accuracy": 22.61904761904762,
      "grounded_hallucination_pct": 0,
      "cost_usd": 0.7106057999999995
    },
    {
      "track": "closed",
      "model": "claude-sonnet-5",
      "provider": "openrouter",
      "total": 249,
      "correct": 177,
      "incorrect": 6,
      "abstained": 66,
      "no_answer": 0,
      "errors": 0,
      "score": 165,
      "score_per_100": 66.26506024096386,
      "runs": 3,
      "score_per_100_min": 62.65060240963855,
      "score_per_100_max": 71.08433734939759,
      "reasoning_correct": 162,
      "reasoning_total": 165,
      "grounded_correct": 15,
      "grounded_total": 84,
      "grounded_wrong": 3,
      "reasoning_accuracy": 98.18181818181819,
      "grounded_accuracy": 17.857142857142858,
      "grounded_hallucination_pct": 16.666666666666668,
      "cost_usd": 0.33236087000000036
    },
    {
      "track": "closed",
      "model": "deepseek-v4-pro",
      "provider": "openrouter",
      "total": 249,
      "correct": 171,
      "incorrect": 14,
      "abstained": 60,
      "no_answer": 4,
      "errors": 0,
      "score": 143,
      "score_per_100": 57.429718875502004,
      "runs": 3,
      "score_per_100_min": 50.602409638554214,
      "score_per_100_max": 62.65060240963855,
      "reasoning_correct": 159,
      "reasoning_total": 164,
      "grounded_correct": 12,
      "grounded_total": 81,
      "grounded_wrong": 9,
      "reasoning_accuracy": 96.95121951219512,
      "grounded_accuracy": 14.814814814814815,
      "grounded_hallucination_pct": 42.857142857142854,
      "cost_usd": 0.15744355800000012
    },
    {
      "track": "closed",
      "model": "gemini-3.1-pro",
      "provider": "openrouter",
      "total": 249,
      "correct": 173,
      "incorrect": 0,
      "abstained": 76,
      "no_answer": 0,
      "errors": 0,
      "score": 173,
      "score_per_100": 69.47791164658635,
      "runs": 3,
      "score_per_100_min": 68.67469879518072,
      "score_per_100_max": 69.87951807228916,
      "reasoning_correct": 165,
      "reasoning_total": 165,
      "grounded_correct": 8,
      "grounded_total": 84,
      "grounded_wrong": 0,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 9.523809523809524,
      "grounded_hallucination_pct": 0,
      "cost_usd": 2.240393779999999
    },
    {
      "track": "closed",
      "model": "gemini-3.5-flash",
      "provider": "openrouter",
      "total": 249,
      "correct": 180,
      "incorrect": 0,
      "abstained": 69,
      "no_answer": 0,
      "errors": 0,
      "score": 180,
      "score_per_100": 72.28915662650603,
      "runs": 3,
      "score_per_100_min": 71.08433734939759,
      "score_per_100_max": 73.49397590361446,
      "reasoning_correct": 165,
      "reasoning_total": 165,
      "grounded_correct": 15,
      "grounded_total": 84,
      "grounded_wrong": 0,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 17.857142857142858,
      "grounded_hallucination_pct": 0,
      "cost_usd": 1.5480489750000004
    },
    {
      "track": "closed",
      "model": "gpt-5.4",
      "provider": "openai",
      "total": 249,
      "correct": 190,
      "incorrect": 44,
      "abstained": 15,
      "no_answer": 0,
      "errors": 0,
      "score": 102,
      "score_per_100": 40.963855421686745,
      "runs": 3,
      "score_per_100_min": 38.55421686746988,
      "score_per_100_max": 43.373493975903614,
      "reasoning_correct": 163,
      "reasoning_total": 165,
      "grounded_correct": 27,
      "grounded_total": 84,
      "grounded_wrong": 42,
      "reasoning_accuracy": 98.78787878787878,
      "grounded_accuracy": 32.142857142857146,
      "grounded_hallucination_pct": 60.869565217391305,
      "cost_usd": 0.24520499999999992
    },
    {
      "track": "closed",
      "model": "gpt-5.4-mini",
      "provider": "openai",
      "total": 249,
      "correct": 174,
      "incorrect": 36,
      "abstained": 39,
      "no_answer": 0,
      "errors": 0,
      "score": 102,
      "score_per_100": 40.963855421686745,
      "runs": 3,
      "score_per_100_min": 33.734939759036145,
      "score_per_100_max": 46.98795180722892,
      "reasoning_correct": 158,
      "reasoning_total": 165,
      "grounded_correct": 16,
      "grounded_total": 84,
      "grounded_wrong": 30,
      "reasoning_accuracy": 95.75757575757575,
      "grounded_accuracy": 19.047619047619047,
      "grounded_hallucination_pct": 65.21739130434783,
      "cost_usd": 0.06355349999999999
    },
    {
      "track": "closed",
      "model": "grok-4.5",
      "provider": "xai",
      "total": 249,
      "correct": 167,
      "incorrect": 1,
      "abstained": 75,
      "no_answer": 6,
      "errors": 0,
      "score": 165,
      "score_per_100": 66.26506024096386,
      "runs": 3,
      "score_per_100_min": 65.06024096385542,
      "score_per_100_max": 67.46987951807229,
      "reasoning_correct": 158,
      "reasoning_total": 159,
      "grounded_correct": 9,
      "grounded_total": 84,
      "grounded_wrong": 0,
      "reasoning_accuracy": 99.37106918238993,
      "grounded_accuracy": 10.714285714285714,
      "grounded_hallucination_pct": 0,
      "cost_usd": 0.22443600000000008
    },
    {
      "track": "closed",
      "model": "kimi-k2.6",
      "provider": "openrouter",
      "total": 249,
      "correct": 160,
      "incorrect": 0,
      "abstained": 75,
      "no_answer": 14,
      "errors": 0,
      "score": 160,
      "score_per_100": 64.2570281124498,
      "runs": 3,
      "score_per_100_min": 62.65060240963855,
      "score_per_100_max": 65.06024096385542,
      "reasoning_correct": 155,
      "reasoning_total": 155,
      "grounded_correct": 5,
      "grounded_total": 80,
      "grounded_wrong": 0,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 6.25,
      "grounded_hallucination_pct": 0,
      "cost_usd": 1.5783579275749997
    },
    {
      "track": "closed",
      "model": "llama-4-maverick",
      "provider": "openrouter",
      "total": 249,
      "correct": 162,
      "incorrect": 16,
      "abstained": 71,
      "no_answer": 0,
      "errors": 0,
      "score": 130,
      "score_per_100": 52.208835341365464,
      "runs": 3,
      "score_per_100_min": 50.602409638554214,
      "score_per_100_max": 53.01204819277108,
      "reasoning_correct": 155,
      "reasoning_total": 165,
      "grounded_correct": 7,
      "grounded_total": 84,
      "grounded_wrong": 6,
      "reasoning_accuracy": 93.93939393939394,
      "grounded_accuracy": 8.333333333333334,
      "grounded_hallucination_pct": 46.15384615384615,
      "cost_usd": 0.03762536174999999
    },
    {
      "track": "closed",
      "model": "qwen3-max",
      "provider": "openrouter",
      "total": 249,
      "correct": 176,
      "incorrect": 8,
      "abstained": 65,
      "no_answer": 0,
      "errors": 0,
      "score": 160,
      "score_per_100": 64.2570281124498,
      "runs": 3,
      "score_per_100_min": 56.626506024096386,
      "score_per_100_max": 69.87951807228916,
      "reasoning_correct": 164,
      "reasoning_total": 165,
      "grounded_correct": 12,
      "grounded_total": 84,
      "grounded_wrong": 7,
      "reasoning_accuracy": 99.39393939393939,
      "grounded_accuracy": 14.285714285714286,
      "grounded_hallucination_pct": 36.8421052631579,
      "cost_usd": 0.2258613630000001
    },
    {
      "track": "mcp",
      "model": "claude-fable-5",
      "provider": "openrouter",
      "total": 249,
      "correct": 208,
      "incorrect": 26,
      "abstained": 15,
      "no_answer": 0,
      "errors": 0,
      "score": 156,
      "score_per_100": 62.65060240963855,
      "runs": 3,
      "score_per_100_min": 56.626506024096386,
      "score_per_100_max": 66.26506024096386,
      "reasoning_correct": 162,
      "reasoning_total": 165,
      "grounded_correct": 46,
      "grounded_total": 84,
      "grounded_wrong": 23,
      "reasoning_accuracy": 98.18181818181819,
      "grounded_accuracy": 54.76190476190476,
      "grounded_hallucination_pct": 33.333333333333336,
      "cost_usd": 41.91691817999998
    },
    {
      "track": "mcp",
      "model": "claude-haiku-4.5",
      "provider": "openrouter",
      "total": 249,
      "correct": 128,
      "incorrect": 19,
      "abstained": 102,
      "no_answer": 0,
      "errors": 0,
      "score": 90,
      "score_per_100": 36.144578313253014,
      "runs": 3,
      "score_per_100_min": 33.734939759036145,
      "score_per_100_max": 38.55421686746988,
      "reasoning_correct": 105,
      "reasoning_total": 165,
      "grounded_correct": 23,
      "grounded_total": 84,
      "grounded_wrong": 15,
      "reasoning_accuracy": 63.63636363636363,
      "grounded_accuracy": 27.38095238095238,
      "grounded_hallucination_pct": 39.473684210526315,
      "cost_usd": 3.8713689200000023
    },
    {
      "track": "mcp",
      "model": "claude-opus-4.8",
      "provider": "openrouter",
      "total": 83,
      "correct": 66,
      "incorrect": 5,
      "abstained": 12,
      "no_answer": 0,
      "errors": 0,
      "score": 56,
      "score_per_100": 67.46987951807229,
      "runs": 1,
      "reasoning_correct": 55,
      "reasoning_total": 55,
      "grounded_correct": 11,
      "grounded_total": 28,
      "grounded_wrong": 5,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 39.285714285714285,
      "grounded_hallucination_pct": 31.25,
      "cost_usd": 2.3833547199999985
    },
    {
      "track": "mcp",
      "model": "claude-sonnet-5",
      "provider": "openrouter",
      "total": 249,
      "correct": 193,
      "incorrect": 23,
      "abstained": 33,
      "no_answer": 0,
      "errors": 0,
      "score": 147,
      "score_per_100": 59.036144578313255,
      "runs": 3,
      "score_per_100_min": 56.626506024096386,
      "score_per_100_max": 61.44578313253012,
      "reasoning_correct": 163,
      "reasoning_total": 165,
      "grounded_correct": 30,
      "grounded_total": 84,
      "grounded_wrong": 21,
      "reasoning_accuracy": 98.78787878787878,
      "grounded_accuracy": 35.714285714285715,
      "grounded_hallucination_pct": 41.1764705882353,
      "cost_usd": 7.908874176
    },
    {
      "track": "mcp",
      "model": "deepseek-v4-pro",
      "provider": "openrouter",
      "total": 83,
      "correct": 56,
      "incorrect": 5,
      "abstained": 20,
      "no_answer": 2,
      "errors": 0,
      "score": 46,
      "score_per_100": 55.42168674698795,
      "runs": 1,
      "reasoning_correct": 48,
      "reasoning_total": 54,
      "grounded_correct": 8,
      "grounded_total": 27,
      "grounded_wrong": 5,
      "reasoning_accuracy": 88.88888888888889,
      "grounded_accuracy": 29.62962962962963,
      "grounded_hallucination_pct": 38.46153846153846,
      "cost_usd": 1.4950787015499991
    },
    {
      "track": "mcp",
      "model": "gemini-3.1-pro",
      "provider": "openrouter",
      "total": 83,
      "correct": 65,
      "incorrect": 3,
      "abstained": 15,
      "no_answer": 0,
      "errors": 0,
      "score": 59,
      "score_per_100": 71.08433734939759,
      "runs": 1,
      "reasoning_correct": 55,
      "reasoning_total": 55,
      "grounded_correct": 10,
      "grounded_total": 28,
      "grounded_wrong": 3,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 35.714285714285715,
      "grounded_hallucination_pct": 23.076923076923077,
      "cost_usd": 6.254359137499998
    },
    {
      "track": "mcp",
      "model": "gemini-3.5-flash",
      "provider": "openrouter",
      "total": 249,
      "correct": 197,
      "incorrect": 7,
      "abstained": 41,
      "no_answer": 4,
      "errors": 0,
      "score": 183,
      "score_per_100": 73.49397590361446,
      "runs": 3,
      "score_per_100_min": 69.87951807228916,
      "score_per_100_max": 78.3132530120482,
      "reasoning_correct": 165,
      "reasoning_total": 165,
      "grounded_correct": 32,
      "grounded_total": 80,
      "grounded_wrong": 7,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 40,
      "grounded_hallucination_pct": 17.94871794871795,
      "cost_usd": 11.613377623124997
    },
    {
      "track": "mcp",
      "model": "gpt-5.4",
      "provider": "openai",
      "total": 249,
      "correct": 161,
      "incorrect": 16,
      "abstained": 72,
      "no_answer": 0,
      "errors": 0,
      "score": 129,
      "score_per_100": 51.80722891566265,
      "runs": 3,
      "score_per_100_min": 44.57831325301205,
      "score_per_100_max": 55.42168674698795,
      "reasoning_correct": 141,
      "reasoning_total": 165,
      "grounded_correct": 20,
      "grounded_total": 84,
      "grounded_wrong": 12,
      "reasoning_accuracy": 85.45454545454545,
      "grounded_accuracy": 23.80952380952381,
      "grounded_hallucination_pct": 37.5,
      "cost_usd": 4.468034499999999
    },
    {
      "track": "mcp",
      "model": "gpt-5.4-mini",
      "provider": "openai",
      "total": 249,
      "correct": 122,
      "incorrect": 27,
      "abstained": 97,
      "no_answer": 3,
      "errors": 0,
      "score": 68,
      "score_per_100": 27.309236947791163,
      "runs": 3,
      "score_per_100_min": 26.50602409638554,
      "score_per_100_max": 28.91566265060241,
      "reasoning_correct": 109,
      "reasoning_total": 162,
      "grounded_correct": 13,
      "grounded_total": 84,
      "grounded_wrong": 17,
      "reasoning_accuracy": 67.28395061728395,
      "grounded_accuracy": 15.476190476190476,
      "grounded_hallucination_pct": 56.666666666666664,
      "cost_usd": 1.0289961000000005
    },
    {
      "track": "mcp",
      "model": "grok-4.5",
      "provider": "xai",
      "total": 249,
      "correct": 171,
      "incorrect": 11,
      "abstained": 66,
      "no_answer": 1,
      "errors": 0,
      "score": 149,
      "score_per_100": 59.83935742971887,
      "runs": 3,
      "score_per_100_min": 51.80722891566265,
      "score_per_100_max": 71.08433734939759,
      "reasoning_correct": 134,
      "reasoning_total": 164,
      "grounded_correct": 37,
      "grounded_total": 84,
      "grounded_wrong": 9,
      "reasoning_accuracy": 81.70731707317073,
      "grounded_accuracy": 44.04761904761905,
      "grounded_hallucination_pct": 19.565217391304348,
      "cost_usd": 15.986348000000001
    },
    {
      "track": "mcp",
      "model": "kimi-k2.6",
      "provider": "openrouter",
      "total": 83,
      "correct": 39,
      "incorrect": 2,
      "abstained": 36,
      "no_answer": 6,
      "errors": 0,
      "score": 35,
      "score_per_100": 42.16867469879518,
      "runs": 1,
      "reasoning_correct": 35,
      "reasoning_total": 55,
      "grounded_correct": 4,
      "grounded_total": 22,
      "grounded_wrong": 2,
      "reasoning_accuracy": 63.63636363636363,
      "grounded_accuracy": 18.181818181818183,
      "grounded_hallucination_pct": 33.333333333333336,
      "cost_usd": 2.8269320442000003
    },
    {
      "track": "mcp",
      "model": "llama-4-maverick",
      "provider": "openrouter",
      "total": 83,
      "correct": 45,
      "incorrect": 18,
      "abstained": 12,
      "no_answer": 8,
      "errors": 0,
      "score": 9,
      "score_per_100": 10.843373493975903,
      "runs": 1,
      "reasoning_correct": 42,
      "reasoning_total": 49,
      "grounded_correct": 3,
      "grounded_total": 26,
      "grounded_wrong": 14,
      "reasoning_accuracy": 85.71428571428571,
      "grounded_accuracy": 11.538461538461538,
      "grounded_hallucination_pct": 82.3529411764706,
      "cost_usd": 0.12412971750000001
    },
    {
      "track": "mcp",
      "model": "qwen3-max",
      "provider": "openrouter",
      "total": 249,
      "correct": 165,
      "incorrect": 30,
      "abstained": 52,
      "no_answer": 2,
      "errors": 0,
      "score": 105,
      "score_per_100": 42.16867469879518,
      "runs": 3,
      "score_per_100_min": 40.963855421686745,
      "score_per_100_max": 43.373493975903614,
      "reasoning_correct": 144,
      "reasoning_total": 163,
      "grounded_correct": 21,
      "grounded_total": 84,
      "grounded_wrong": 29,
      "reasoning_accuracy": 88.34355828220859,
      "grounded_accuracy": 25,
      "grounded_hallucination_pct": 58,
      "cost_usd": 4.4390994882
    }
  ],
  "gaps": [
    {
      "model": "claude-fable-5",
      "grounded_acc_closed": 26.19047619047619,
      "grounded_acc_mcp": 54.76190476190476,
      "halluc_closed": 15.384615384615385,
      "halluc_mcp": 33.333333333333336
    },
    {
      "model": "claude-haiku-4.5",
      "grounded_acc_closed": 8.333333333333334,
      "grounded_acc_mcp": 27.38095238095238,
      "halluc_closed": 50,
      "halluc_mcp": 39.473684210526315
    },
    {
      "model": "claude-opus-4.8",
      "grounded_acc_closed": 22.61904761904762,
      "grounded_acc_mcp": 39.285714285714285,
      "halluc_closed": 0,
      "halluc_mcp": 31.25
    },
    {
      "model": "claude-sonnet-5",
      "grounded_acc_closed": 17.857142857142858,
      "grounded_acc_mcp": 35.714285714285715,
      "halluc_closed": 16.666666666666668,
      "halluc_mcp": 41.1764705882353
    },
    {
      "model": "deepseek-v4-pro",
      "grounded_acc_closed": 14.814814814814815,
      "grounded_acc_mcp": 29.62962962962963,
      "halluc_closed": 42.857142857142854,
      "halluc_mcp": 38.46153846153846
    },
    {
      "model": "gemini-3.1-pro",
      "grounded_acc_closed": 9.523809523809524,
      "grounded_acc_mcp": 35.714285714285715,
      "halluc_closed": 0,
      "halluc_mcp": 23.076923076923077
    },
    {
      "model": "gemini-3.5-flash",
      "grounded_acc_closed": 17.857142857142858,
      "grounded_acc_mcp": 40,
      "halluc_closed": 0,
      "halluc_mcp": 17.94871794871795
    },
    {
      "model": "gpt-5.4",
      "grounded_acc_closed": 32.142857142857146,
      "grounded_acc_mcp": 23.80952380952381,
      "halluc_closed": 60.869565217391305,
      "halluc_mcp": 37.5
    },
    {
      "model": "gpt-5.4-mini",
      "grounded_acc_closed": 19.047619047619047,
      "grounded_acc_mcp": 15.476190476190476,
      "halluc_closed": 65.21739130434783,
      "halluc_mcp": 56.666666666666664
    },
    {
      "model": "grok-4.5",
      "grounded_acc_closed": 10.714285714285714,
      "grounded_acc_mcp": 44.04761904761905,
      "halluc_closed": 0,
      "halluc_mcp": 19.565217391304348
    },
    {
      "model": "kimi-k2.6",
      "grounded_acc_closed": 6.25,
      "grounded_acc_mcp": 18.181818181818183,
      "halluc_closed": 0,
      "halluc_mcp": 33.333333333333336
    },
    {
      "model": "llama-4-maverick",
      "grounded_acc_closed": 8.333333333333334,
      "grounded_acc_mcp": 11.538461538461538,
      "halluc_closed": 46.15384615384615,
      "halluc_mcp": 82.3529411764706
    },
    {
      "model": "qwen3-max",
      "grounded_acc_closed": 14.285714285714286,
      "grounded_acc_mcp": 25,
      "halluc_closed": 36.8421052631579,
      "halluc_mcp": 58
    }
  ]
}