{ "corpus_fingerprint": "278-8bc1c4f8addd", "revision": "2026-10-17", "published_at": "2026-10-17T01:12:58Z", "grader_version": "2026-10-10", "rows": [ { "track": "closed", "model": "claude-opus-5.5", "provider": "claude-code", "total": 276, "correct": 132, "incorrect": 25, "abstained": 119, "no_answer": 0, "errors": 0, "truncated": 0, "score": 82, "score_per_100": 29.71014492753623, "runs": 1, "reasoning_correct": 42, "reasoning_total": 43, "grounded_correct": 90, "grounded_total": 233, "grounded_wrong": 24, "reasoning_accuracy": 97.67441860465117, "grounded_accuracy": 38.62660944206009, "grounded_hallucination_pct": 21.05263157894737, "cost_usd": 4.491024000000002 }, { "track": "closed", "model": "claude-fable-5.1", "provider": "claude-code", "total": 276, "correct": 139, "incorrect": 30, "abstained": 107, "no_answer": 0, "errors": 0, "truncated": 0, "score": 79, "score_per_100": 28.6231884057971, "runs": 1, "reasoning_correct": 43, "reasoning_total": 43, "grounded_correct": 96, "grounded_total": 233, "grounded_wrong": 30, "reasoning_accuracy": 100, "grounded_accuracy": 41.201716738197426, "grounded_hallucination_pct": 23.80952380952381, "cost_usd": 13.56906249999999 }, { "track": "closed", "model": "grok-4.7", "provider": "xai", "total": 276, "correct": 85, "incorrect": 5, "abstained": 186, "no_answer": 0, "errors": 0, "truncated": 0, "score": 75, "score_per_100": 27.17391304347826, "runs": 1, "reasoning_correct": 39, "reasoning_total": 43, "grounded_correct": 46, "grounded_total": 233, "grounded_wrong": 2, "reasoning_accuracy": 90.69767441860465, "grounded_accuracy": 19.742489270386265, "grounded_hallucination_pct": 4.166666666666667, "cost_usd": 2.978921999999998 }, { "track": "closed", "model": "claude-sonnet-5.5", "provider": "claude-code", "total": 276, "correct": 128, "incorrect": 45, "abstained": 103, "no_answer": 0, "errors": 0, "truncated": 0, "score": 38, "score_per_100": 13.768115942028986, "runs": 1, "reasoning_correct": 43, "reasoning_total": 43, "grounded_correct": 85, "grounded_total": 233, "grounded_wrong": 45, "reasoning_accuracy": 100, "grounded_accuracy": 36.48068669527897, "grounded_hallucination_pct": 34.61538461538461, "cost_usd": 2.2146999999999992 }, { "track": "closed", "model": "claude-haiku-5.5", "provider": "claude-code", "total": 276, "correct": 77, "incorrect": 32, "abstained": 167, "no_answer": 0, "errors": 0, "truncated": 0, "score": 13, "score_per_100": 4.7101449275362315, "runs": 1, "reasoning_correct": 41, "reasoning_total": 43, "grounded_correct": 36, "grounded_total": 233, "grounded_wrong": 30, "reasoning_accuracy": 95.34883720930233, "grounded_accuracy": 15.450643776824034, "grounded_hallucination_pct": 45.45454545454545, "cost_usd": 0.1267466000000001 }, { "track": "closed", "model": "gemma4-12b", "provider": "ollama", "total": 276, "correct": 30, "incorrect": 102, "abstained": 105, "no_answer": 0, "errors": 0, "truncated": 39, "score": -174, "score_per_100": -73.41772151898734, "runs": 1, "reasoning_correct": 22, "reasoning_total": 30, "grounded_correct": 8, "grounded_total": 207, "grounded_wrong": 94, "reasoning_accuracy": 73.33333333333333, "grounded_accuracy": 3.864734299516908, "grounded_hallucination_pct": 92.15686274509804, "cost_usd": 0 }, { "track": "closed", "model": "gpt-oss-20b", "provider": "ollama", "total": 276, "correct": 40, "incorrect": 145, "abstained": 88, "no_answer": 2, "errors": 0, "truncated": 1, "score": -250, "score_per_100": -90.9090909090909, "runs": 1, "reasoning_correct": 27, "reasoning_total": 40, "grounded_correct": 13, "grounded_total": 233, "grounded_wrong": 132, "reasoning_accuracy": 67.5, "grounded_accuracy": 5.579399141630901, "grounded_hallucination_pct": 91.03448275862068, "cost_usd": 0 }, { "track": "closed", "model": "deepseek-r1-14b", "provider": "ollama", "total": 276, "correct": 34, "incorrect": 145, "abstained": 96, "no_answer": 0, "errors": 0, "truncated": 1, "score": -256, "score_per_100": -93.0909090909091, "runs": 1, "reasoning_correct": 22, "reasoning_total": 42, "grounded_correct": 12, "grounded_total": 233, "grounded_wrong": 125, "reasoning_accuracy": 52.38095238095238, "grounded_accuracy": 5.150214592274678, "grounded_hallucination_pct": 91.24087591240875, "cost_usd": 0 }, { "track": "mcp", "model": "claude-opus-5.5", "provider": "claude-code", "total": 828, "correct": 717, "incorrect": 13, "abstained": 37, "no_answer": 61, "errors": 0, "truncated": 0, "score": 691, "score_per_100": 83.45410628019323, "runs": 3, "score_per_100_min": 82.2463768115942, "score_per_100_max": 84.78260869565217, "reasoning_correct": 126, "reasoning_total": 129, "grounded_correct": 591, "grounded_total": 638, "grounded_wrong": 10, "reasoning_accuracy": 97.67441860465117, "grounded_accuracy": 92.6332288401254, "grounded_hallucination_pct": 1.663893510815308, "cost_usd": 109.30653860000002 }, { "track": "mcp", "model": "claude-fable-5.1", "provider": "claude-code", "total": 552, "correct": 489, "incorrect": 18, "abstained": 31, "no_answer": 14, "errors": 0, "truncated": 0, "score": 453, "score_per_100": 82.06521739130434, "runs": 2, "score_per_100_min": 81.8840579710145, "score_per_100_max": 82.2463768115942, "reasoning_correct": 84, "reasoning_total": 86, "grounded_correct": 405, "grounded_total": 452, "grounded_wrong": 16, "reasoning_accuracy": 97.67441860465117, "grounded_accuracy": 89.60176991150442, "grounded_hallucination_pct": 3.8004750593824226, "cost_usd": 305.61237250000005 }, { "track": "mcp", "model": "claude-sonnet-5.5", "provider": "claude-code", "total": 828, "correct": 699, "incorrect": 48, "abstained": 47, "no_answer": 34, "errors": 0, "truncated": 0, "score": 603, "score_per_100": 72.82608695652173, "runs": 3, "score_per_100_min": 70.65217391304348, "score_per_100_max": 74.27536231884058, "reasoning_correct": 121, "reasoning_total": 129, "grounded_correct": 578, "grounded_total": 665, "grounded_wrong": 40, "reasoning_accuracy": 93.7984496124031, "grounded_accuracy": 86.9172932330827, "grounded_hallucination_pct": 6.472491909385114, "cost_usd": 43.64014809999996 }, { "track": "mcp", "model": "grok-4.7", "provider": "xai", "total": 552, "correct": 436, "incorrect": 21, "abstained": 95, "no_answer": 0, "errors": 0, "truncated": 0, "score": 394, "score_per_100": 71.3768115942029, "runs": 2, "score_per_100_min": 70.65217391304348, "score_per_100_max": 72.10144927536231, "reasoning_correct": 80, "reasoning_total": 86, "grounded_correct": 356, "grounded_total": 466, "grounded_wrong": 17, "reasoning_accuracy": 93.02325581395348, "grounded_accuracy": 76.39484978540773, "grounded_hallucination_pct": 4.557640750670242, "cost_usd": 54.312758000000045 }, { "track": "mcp", "model": "claude-haiku-5.5", "provider": "claude-code", "total": 828, "correct": 629, "incorrect": 55, "abstained": 99, "no_answer": 45, "errors": 0, "truncated": 0, "score": 519, "score_per_100": 62.68115942028985, "runs": 3, "score_per_100_min": 60.507246376811594, "score_per_100_max": 65.21739130434783, "reasoning_correct": 119, "reasoning_total": 129, "grounded_correct": 510, "grounded_total": 654, "grounded_wrong": 48, "reasoning_accuracy": 92.24806201550388, "grounded_accuracy": 77.98165137614679, "grounded_hallucination_pct": 8.602150537634408, "cost_usd": 4.191941170000002 }, { "track": "mcp", "model": "gpt-oss-20b", "provider": "ollama", "total": 828, "correct": 383, "incorrect": 173, "abstained": 44, "no_answer": 228, "errors": 0, "truncated": 0, "score": 37, "score_per_100": 4.468599033816425, "runs": 3, "score_per_100_min": 2.536231884057971, "score_per_100_max": 5.797101449275362, "reasoning_correct": 79, "reasoning_total": 107, "grounded_correct": 304, "grounded_total": 493, "grounded_wrong": 146, "reasoning_accuracy": 73.83177570093459, "grounded_accuracy": 61.66328600405679, "grounded_hallucination_pct": 32.44444444444444, "cost_usd": 0 }, { "track": "mcp", "model": "gemma4-12b", "provider": "ollama", "total": 828, "correct": 400, "incorrect": 214, "abstained": 186, "no_answer": 2, "errors": 0, "truncated": 26, "score": -28, "score_per_100": -3.491271820448878, "runs": 3, "score_per_100_min": -6.7164179104477615, "score_per_100_max": 1.8656716417910448, "reasoning_correct": 83, "reasoning_total": 118, "grounded_correct": 317, "grounded_total": 682, "grounded_wrong": 192, "reasoning_accuracy": 70.33898305084746, "grounded_accuracy": 46.48093841642229, "grounded_hallucination_pct": 37.721021611001966, "cost_usd": 0 } ], "gaps": [ { "model": "claude-fable-5.1", "score_gap_per_100": 53.44202898550724, "grounded_acc_closed": 41.201716738197426, "grounded_acc_mcp": 89.60176991150442, "halluc_closed": 23.80952380952381, "halluc_mcp": 3.8004750593824226 }, { "model": "claude-haiku-5.5", "score_gap_per_100": 57.97101449275362, "grounded_acc_closed": 15.450643776824034, "grounded_acc_mcp": 77.98165137614679, "halluc_closed": 45.45454545454545, "halluc_mcp": 8.602150537634408 }, { "model": "claude-opus-5.5", "score_gap_per_100": 53.743961352656996, "grounded_acc_closed": 38.62660944206009, "grounded_acc_mcp": 92.6332288401254, "halluc_closed": 21.05263157894737, "halluc_mcp": 1.663893510815308 }, { "model": "claude-sonnet-5.5", "score_gap_per_100": 59.05797101449275, "grounded_acc_closed": 36.48068669527897, "grounded_acc_mcp": 86.9172932330827, "halluc_closed": 34.61538461538461, "halluc_mcp": 6.472491909385114 }, { "model": "gemma4-12b", "score_gap_per_100": 69.92644969853846, "grounded_acc_closed": 3.864734299516908, "grounded_acc_mcp": 46.48093841642229, "halluc_closed": 92.15686274509804, "halluc_mcp": 37.721021611001966 }, { "model": "gpt-oss-20b", "score_gap_per_100": 95.37768994290734, "grounded_acc_closed": 5.579399141630901, "grounded_acc_mcp": 61.66328600405679, "halluc_closed": 91.03448275862068, "halluc_mcp": 32.44444444444444 }, { "model": "grok-4.7", "score_gap_per_100": 44.20289855072464, "grounded_acc_closed": 19.742489270386265, "grounded_acc_mcp": 76.39484978540773, "halluc_closed": 4.166666666666667, "halluc_mcp": 4.557640750670242 } ], "jobs": [ "baseline", "catalog", "constellation", "debris", "ground", "market", "regulatory" ], "errata": [ { "id": "con-galileo-launched", "action": "rekey", "value": "34", "reason": "The question counts IOV and FOC satellites; the ground-truth SQL matched only names starting \"GALILEO \" and missed the 4 IOV satellites, giving 30. 34 Galileo satellites had been launched (latest 2025-12-17).", "date": "2026-10-11" }, { "id": "deb-atlas5-centaur-afspc11-debris", "action": "drop", "reason": "The question names the wrong launch. The Centaur that fragmented in September 2024 is 2018-079B from the AEHF-4 launch (2018-10-17, 214 debris cataloged). AFSPC-11 (2018-036) has 1 debris object, which is what models answered.", "date": "2026-10-11" }, { "id": "reg-ngso-market-access-count", "action": "rekey", "value": "9", "reason": "The question asks for foreign-licensed NGSO systems with US market access; the ground-truth SQL counted every granted NGSO system, U.S.-licensed ones included, giving 52. Excluding a U.S. licensing administration leaves 9 (Kepler, Telesat Ka-band, ExactView-1, ExactView-7, Kineis, Arctic Satellite Broadband Mission, two OneWeb systems, O3B).", "date": "2026-10-11" }, { "id": "reg-noaa-remote-sensing-licensees", "action": "drop", "reason": "The question asks how many companies currently hold a license, but the answer key comes from a NOAA CRSRA licensee snapshot last seen 2024-07-27; no current list was available to verify it. To be restated as an as-of question, or rekeyed from a current list, in the next corpus.", "date": "2026-10-11" } ], "verdicts": [ { "model": "claude-fable-5.1", "track": "closed", "job": "baseline", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 25, "gradeable": 25, "correct": 25, "incorrect": 0, "abstained": 0, "no_answer": 0, "correct_pct": 1, "wrong_pct": 0, "correct_lo": 0.8668035060468213, "correct_hi": 1, "wrong_lo": 0, "wrong_hi": 0.13319649395317873, "verdict": "trust" }, { "model": "claude-fable-5.1", "track": "mcp", "job": "baseline", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 2, "questions": 25, "gradeable": 50, "correct": 50, "incorrect": 0, "abstained": 0, "no_answer": 0, "correct_pct": 1, "wrong_pct": 0, "correct_lo": 0.8668035060468213, "correct_hi": 1, "wrong_lo": 0, "wrong_hi": 0.13319649395317873, "verdict": "trust" }, { "model": "claude-fable-5.1", "track": "closed", "job": "catalog", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 45, "gradeable": 45, "correct": 29, "incorrect": 3, "abstained": 13, "no_answer": 0, "correct_pct": 0.6444444444444445, "wrong_pct": 0.06666666666666667, "correct_lo": 0.49835546011952203, "correct_hi": 0.7678110864309596, "wrong_lo": 0.0229316164621471, "wrong_hi": 0.17856874388640825, "verdict": "verify" }, { "model": "claude-fable-5.1", "track": "mcp", "job": "catalog", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 2, "questions": 45, "gradeable": 90, "correct": 89, "incorrect": 1, "abstained": 0, "no_answer": 0, "correct_pct": 0.9888888888888889, "wrong_pct": 0.011111111111111112, "correct_lo": 0.9020324502033684, "correct_hi": 0.9988389381213386, "wrong_lo": 0.0011610618786614471, "wrong_hi": 0.09796754979663178, "verdict": "trust" }, { "model": "claude-fable-5.1", "track": "closed", "job": "constellation", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 43, "gradeable": 43, "correct": 23, "incorrect": 9, "abstained": 11, "no_answer": 0, "correct_pct": 0.5348837209302325, "wrong_pct": 0.20930232558139536, "correct_lo": 0.38915389728872773, "correct_hi": 0.6748917373608144, "wrong_lo": 0.11422705997811457, "wrong_hi": 0.35205931794236645, "verdict": "dont_use" }, { "model": "claude-fable-5.1", "track": "mcp", "job": "constellation", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 2, "questions": 43, "gradeable": 86, "correct": 86, "incorrect": 0, "abstained": 0, "no_answer": 0, "correct_pct": 1, "wrong_pct": 0, "correct_lo": 0.9179874299767729, "correct_hi": 1, "wrong_lo": 0, "wrong_hi": 0.08201257002322722, "verdict": "trust" }, { "model": "claude-fable-5.1", "track": "closed", "job": "debris", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 42, "gradeable": 42, "correct": 19, "incorrect": 4, "abstained": 19, "no_answer": 0, "correct_pct": 0.4523809523809524, "wrong_pct": 0.09523809523809523, "correct_lo": 0.3122316381863852, "correct_hi": 0.6005113725287732, "wrong_lo": 0.03766163882831211, "wrong_hi": 0.22065395225053333, "verdict": "verify" }, { "model": "claude-fable-5.1", "track": "mcp", "job": "debris", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 2, "questions": 42, "gradeable": 84, "correct": 79, "incorrect": 2, "abstained": 3, "no_answer": 0, "correct_pct": 0.9404761904761905, "wrong_pct": 0.023809523809523808, "correct_lo": 0.8257573017536823, "correct_hi": 0.9813698491311036, "wrong_lo": 0.004215295499100304, "wrong_hi": 0.12321481165248255, "verdict": "trust" }, { "model": "claude-fable-5.1", "track": "closed", "job": "ground", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 22, "gradeable": 22, "correct": 6, "incorrect": 5, "abstained": 11, "no_answer": 0, "correct_pct": 0.2727272727272727, "wrong_pct": 0.22727272727272727, "correct_lo": 0.13150582015814438, "correct_hi": 0.4815212369900198, "wrong_lo": 0.10122890794820358, "wrong_hi": 0.4344035606295935, "verdict": "dont_use" }, { "model": "claude-fable-5.1", "track": "mcp", "job": "ground", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 2, "questions": 22, "gradeable": 44, "correct": 32, "incorrect": 6, "abstained": 2, "no_answer": 4, "correct_pct": 0.7272727272727273, "wrong_pct": 0.13636363636363635, "correct_lo": 0.5184787630099802, "correct_hi": 0.8684941798418556, "wrong_lo": 0.04748917264236269, "wrong_hi": 0.3333541187947, "verdict": "verify" }, { "model": "claude-fable-5.1", "track": "closed", "job": "market", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 41, "gradeable": 41, "correct": 18, "incorrect": 7, "abstained": 16, "no_answer": 0, "correct_pct": 0.43902439024390244, "wrong_pct": 0.17073170731707318, "correct_lo": 0.2988990477210983, "correct_hi": 0.5895973484824271, "wrong_lo": 0.08525143995273901, "wrong_hi": 0.3126290995462977, "verdict": "verify" }, { "model": "claude-fable-5.1", "track": "mcp", "job": "market", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 2, "questions": 41, "gradeable": 82, "correct": 74, "incorrect": 2, "abstained": 1, "no_answer": 5, "correct_pct": 0.9024390243902439, "wrong_pct": 0.024390243902439025, "correct_lo": 0.7745202448096945, "correct_hi": 0.9614035402470386, "wrong_lo": 0.00431842027071648, "wrong_hi": 0.12595347011678087, "verdict": "trust" }, { "model": "claude-fable-5.1", "track": "closed", "job": "regulatory", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 40, "gradeable": 40, "correct": 14, "incorrect": 2, "abstained": 24, "no_answer": 0, "correct_pct": 0.35, "wrong_pct": 0.05, "correct_lo": 0.2213433317104915, "correct_hi": 0.5049440300646263, "wrong_lo": 0.013820377146101098, "wrong_hi": 0.165041708179252, "verdict": "verify" }, { "model": "claude-fable-5.1", "track": "mcp", "job": "regulatory", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 2, "questions": 40, "gradeable": 80, "correct": 47, "incorrect": 3, "abstained": 25, "no_answer": 5, "correct_pct": 0.5875, "wrong_pct": 0.0375, "correct_lo": 0.4339080619817175, "correct_hi": 0.7257576436494639, "wrong_lo": 0.008707641902894095, "wrong_hi": 0.1473450569037188, "verdict": "verify" }, { "model": "claude-fable-5.1", "track": "closed", "kind": "os-derived", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 13, "gradeable": 13, "correct": 0, "incorrect": 0, "abstained": 13, "no_answer": 0, "correct_pct": 0, "wrong_pct": 0, "correct_lo": 0, "correct_hi": 0.22810184305529166, "wrong_lo": 0, "wrong_hi": 0.22810184305529166, "verdict": "verify" }, { "model": "claude-fable-5.1", "track": "mcp", "kind": "os-derived", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 2, "questions": 13, "gradeable": 26, "correct": 22, "incorrect": 4, "abstained": 0, "no_answer": 0, "correct_pct": 0.8461538461538461, "wrong_pct": 0.15384615384615385, "correct_lo": 0.5776482949136899, "correct_hi": 0.9567427368172621, "wrong_lo": 0.043257263182737954, "wrong_hi": 0.42235170508631015, "verdict": "verify" }, { "model": "claude-haiku-5.5", "track": "closed", "job": "baseline", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 25, "gradeable": 25, "correct": 25, "incorrect": 0, "abstained": 0, "no_answer": 0, "correct_pct": 1, "wrong_pct": 0, "correct_lo": 0.8668035060468213, "correct_hi": 1, "wrong_lo": 0, "wrong_hi": 0.13319649395317873, "verdict": "trust" }, { "model": "claude-haiku-5.5", "track": "mcp", "job": "baseline", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 25, "gradeable": 75, "correct": 74, "incorrect": 1, "abstained": 0, "no_answer": 0, "correct_pct": 0.9866666666666667, "wrong_pct": 0.013333333333333334, "correct_lo": 0.8446808865024286, "correct_hi": 0.9990078593831441, "wrong_lo": 0.0009921406168558883, "wrong_hi": 0.15531911349757144, "verdict": "trust" }, { "model": "claude-haiku-5.5", "track": "closed", "job": "catalog", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 45, "gradeable": 45, "correct": 13, "incorrect": 8, "abstained": 24, "no_answer": 0, "correct_pct": 0.28888888888888886, "wrong_pct": 0.17777777777777778, "correct_lo": 0.17729918031900166, "correct_hi": 0.4336881747226023, "wrong_lo": 0.0929427892585383, "wrong_hi": 0.3133010684365414, "verdict": "verify" }, { "model": "claude-haiku-5.5", "track": "mcp", "job": "catalog", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 45, "gradeable": 135, "correct": 123, "incorrect": 2, "abstained": 1, "no_answer": 9, "correct_pct": 0.9111111111111111, "wrong_pct": 0.014814814814814815, "correct_lo": 0.792661488316343, "correct_hi": 0.9648894518657969, "wrong_lo": 0.0019442110036230614, "wrong_hi": 0.10400918391655428, "verdict": "trust" }, { "model": "claude-haiku-5.5", "track": "closed", "job": "constellation", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 43, "gradeable": 43, "correct": 10, "incorrect": 6, "abstained": 27, "no_answer": 0, "correct_pct": 0.23255813953488372, "wrong_pct": 0.13953488372093023, "correct_lo": 0.1315353846924483, "correct_hi": 0.37744808299439414, "wrong_lo": 0.06555634267713947, "wrong_hi": 0.27263876594425684, "verdict": "verify" }, { "model": "claude-haiku-5.5", "track": "mcp", "job": "constellation", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 43, "gradeable": 129, "correct": 106, "incorrect": 16, "abstained": 5, "no_answer": 2, "correct_pct": 0.8217054263565892, "wrong_pct": 0.12403100775193798, "correct_lo": 0.6825766401181178, "correct_hi": 0.9080664349832167, "wrong_lo": 0.05556165364366676, "wrong_hi": 0.2541687284668219, "verdict": "verify" }, { "model": "claude-haiku-5.5", "track": "closed", "job": "debris", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 42, "gradeable": 42, "correct": 6, "incorrect": 6, "abstained": 30, "no_answer": 0, "correct_pct": 0.14285714285714285, "wrong_pct": 0.14285714285714285, "correct_lo": 0.06715875684189936, "correct_hi": 0.2784138235217878, "wrong_lo": 0.06715875684189936, "wrong_hi": 0.2784138235217878, "verdict": "verify" }, { "model": "claude-haiku-5.5", "track": "mcp", "job": "debris", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 42, "gradeable": 126, "correct": 110, "incorrect": 1, "abstained": 13, "no_answer": 2, "correct_pct": 0.873015873015873, "wrong_pct": 0.007936507936507936, "correct_lo": 0.740428746279078, "correct_hi": 0.9430843364521821, "wrong_lo": 0.0005903553473814785, "wrong_hi": 0.09775408870925414, "verdict": "trust" }, { "model": "claude-haiku-5.5", "track": "closed", "job": "ground", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 22, "gradeable": 22, "correct": 1, "incorrect": 2, "abstained": 19, "no_answer": 0, "correct_pct": 0.045454545454545456, "wrong_pct": 0.09090909090909091, "correct_lo": 0.008069219301147859, "correct_hi": 0.21798489499518053, "wrong_lo": 0.025294896282837764, "wrong_hi": 0.27815380658385785, "verdict": "verify" }, { "model": "claude-haiku-5.5", "track": "mcp", "job": "ground", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 22, "gradeable": 66, "correct": 35, "incorrect": 9, "abstained": 15, "no_answer": 7, "correct_pct": 0.5303030303030303, "wrong_pct": 0.13636363636363635, "correct_lo": 0.3333178989198782, "correct_hi": 0.7182784934603665, "wrong_lo": 0.04748917264236269, "wrong_hi": 0.3333541187947, "verdict": "verify" }, { "model": "claude-haiku-5.5", "track": "closed", "job": "market", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 41, "gradeable": 41, "correct": 9, "incorrect": 5, "abstained": 27, "no_answer": 0, "correct_pct": 0.21951219512195122, "wrong_pct": 0.12195121951219512, "correct_lo": 0.12003040155386346, "correct_hi": 0.36705302098235293, "wrong_lo": 0.053232579973408245, "wrong_hi": 0.25544507648844866, "verdict": "verify" }, { "model": "claude-haiku-5.5", "track": "mcp", "job": "market", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 41, "gradeable": 123, "correct": 101, "incorrect": 2, "abstained": 6, "no_answer": 14, "correct_pct": 0.8211382113821138, "wrong_pct": 0.016260162601626018, "correct_lo": 0.6781300384243848, "correct_hi": 0.9091222749037154, "wrong_lo": 0.0021342021336350214, "wrong_hi": 0.11327054108099902, "verdict": "trust" }, { "model": "claude-haiku-5.5", "track": "closed", "job": "regulatory", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 40, "gradeable": 40, "correct": 8, "incorrect": 5, "abstained": 27, "no_answer": 0, "correct_pct": 0.2, "wrong_pct": 0.125, "correct_lo": 0.10499864973166874, "correct_hi": 0.34757607381856676, "wrong_lo": 0.05459419773541284, "wrong_hi": 0.26112420670238146, "verdict": "verify" }, { "model": "claude-haiku-5.5", "track": "mcp", "job": "regulatory", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 40, "gradeable": 120, "correct": 43, "incorrect": 13, "abstained": 53, "no_answer": 11, "correct_pct": 0.35833333333333334, "wrong_pct": 0.10833333333333334, "correct_lo": 0.22826293910901654, "correct_hi": 0.5132306803452614, "wrong_lo": 0.04445863307797926, "wrong_hi": 0.24084725600149476, "verdict": "verify" }, { "model": "claude-haiku-5.5", "track": "closed", "kind": "os-derived", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 13, "gradeable": 13, "correct": 0, "incorrect": 0, "abstained": 13, "no_answer": 0, "correct_pct": 0, "wrong_pct": 0, "correct_lo": 0, "correct_hi": 0.22810184305529166, "wrong_lo": 0, "wrong_hi": 0.22810184305529166, "verdict": "verify" }, { "model": "claude-haiku-5.5", "track": "mcp", "kind": "os-derived", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 13, "gradeable": 39, "correct": 26, "incorrect": 7, "abstained": 6, "no_answer": 0, "correct_pct": 0.6666666666666666, "wrong_pct": 0.1794871794871795, "correct_lo": 0.4003196931375318, "correct_hi": 0.8569796925107044, "wrong_lo": 0.05526981696044017, "wrong_hi": 0.44992367217756735, "verdict": "verify" }, { "model": "claude-opus-5.5", "track": "closed", "job": "baseline", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 25, "gradeable": 25, "correct": 25, "incorrect": 0, "abstained": 0, "no_answer": 0, "correct_pct": 1, "wrong_pct": 0, "correct_lo": 0.8668035060468213, "correct_hi": 1, "wrong_lo": 0, "wrong_hi": 0.13319649395317873, "verdict": "trust" }, { "model": "claude-opus-5.5", "track": "mcp", "job": "baseline", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 25, "gradeable": 75, "correct": 75, "incorrect": 0, "abstained": 0, "no_answer": 0, "correct_pct": 1, "wrong_pct": 0, "correct_lo": 0.8668035060468213, "correct_hi": 1, "wrong_lo": 0, "wrong_hi": 0.13319649395317873, "verdict": "trust" }, { "model": "claude-opus-5.5", "track": "closed", "job": "catalog", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 45, "gradeable": 45, "correct": 30, "incorrect": 5, "abstained": 10, "no_answer": 0, "correct_pct": 0.6666666666666666, "wrong_pct": 0.1111111111111111, "correct_lo": 0.520702168092554, "correct_hi": 0.7864130779272324, "wrong_lo": 0.04840400437825558, "wrong_hi": 0.23499375490890945, "verdict": "verify" }, { "model": "claude-opus-5.5", "track": "mcp", "job": "catalog", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 45, "gradeable": 135, "correct": 135, "incorrect": 0, "abstained": 0, "no_answer": 0, "correct_pct": 1, "wrong_pct": 0, "correct_lo": 0.9213457380593594, "correct_hi": 1, "wrong_lo": 6.938893903907228e-18, "wrong_hi": 0.07865426194064076, "verdict": "trust" }, { "model": "claude-opus-5.5", "track": "closed", "job": "constellation", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 43, "gradeable": 43, "correct": 22, "incorrect": 5, "abstained": 16, "no_answer": 0, "correct_pct": 0.5116279069767442, "wrong_pct": 0.11627906976744186, "correct_lo": 0.36752063766074083, "correct_hi": 0.6538279072224399, "wrong_lo": 0.050703529948620846, "wrong_hi": 0.24479448890641395, "verdict": "verify" }, { "model": "claude-opus-5.5", "track": "mcp", "job": "constellation", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 43, "gradeable": 129, "correct": 129, "incorrect": 0, "abstained": 0, "no_answer": 0, "correct_pct": 1, "wrong_pct": 0, "correct_lo": 0.9179874299767729, "correct_hi": 1, "wrong_lo": 0, "wrong_hi": 0.08201257002322722, "verdict": "trust" }, { "model": "claude-opus-5.5", "track": "closed", "job": "debris", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 42, "gradeable": 42, "correct": 17, "incorrect": 3, "abstained": 22, "no_answer": 0, "correct_pct": 0.40476190476190477, "wrong_pct": 0.07142857142857142, "correct_lo": 0.27042647671209086, "correct_hi": 0.5550595447182258, "wrong_lo": 0.024589963551172878, "wrong_hi": 0.19009713288525168, "verdict": "verify" }, { "model": "claude-opus-5.5", "track": "mcp", "job": "debris", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 42, "gradeable": 126, "correct": 117, "incorrect": 3, "abstained": 6, "no_answer": 0, "correct_pct": 0.9285714285714286, "wrong_pct": 0.023809523809523808, "correct_lo": 0.8099028671147483, "correct_hi": 0.9754100364488272, "wrong_lo": 0.004215295499100304, "wrong_hi": 0.12321481165248255, "verdict": "trust" }, { "model": "claude-opus-5.5", "track": "closed", "job": "ground", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 22, "gradeable": 22, "correct": 4, "incorrect": 5, "abstained": 13, "no_answer": 0, "correct_pct": 0.18181818181818182, "wrong_pct": 0.22727272727272727, "correct_lo": 0.07306766024105121, "correct_hi": 0.38517021976637866, "wrong_lo": 0.10122890794820358, "wrong_hi": 0.4344035606295935, "verdict": "dont_use" }, { "model": "claude-opus-5.5", "track": "mcp", "job": "ground", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 22, "gradeable": 66, "correct": 45, "incorrect": 3, "abstained": 2, "no_answer": 16, "correct_pct": 0.6818181818181818, "wrong_pct": 0.045454545454545456, "correct_lo": 0.47318222782839336, "correct_hi": 0.8363961264530752, "wrong_lo": 0.008069219301147859, "wrong_hi": 0.21798489499518053, "verdict": "verify" }, { "model": "claude-opus-5.5", "track": "closed", "job": "market", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 41, "gradeable": 41, "correct": 15, "incorrect": 6, "abstained": 20, "no_answer": 0, "correct_pct": 0.36585365853658536, "wrong_pct": 0.14634146341463414, "correct_lo": 0.23589630416936816, "correct_hi": 0.5187957674783876, "wrong_lo": 0.06884154529455412, "wrong_hi": 0.28443755268589266, "verdict": "verify" }, { "model": "claude-opus-5.5", "track": "mcp", "job": "market", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 41, "gradeable": 123, "correct": 103, "incorrect": 2, "abstained": 5, "no_answer": 13, "correct_pct": 0.8373983739837398, "wrong_pct": 0.016260162601626018, "correct_lo": 0.6966872591434594, "correct_hi": 0.9202993485303674, "wrong_lo": 0.0021342021336350214, "wrong_hi": 0.11327054108099902, "verdict": "trust" }, { "model": "claude-opus-5.5", "track": "closed", "job": "regulatory", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 40, "gradeable": 40, "correct": 14, "incorrect": 1, "abstained": 25, "no_answer": 0, "correct_pct": 0.35, "wrong_pct": 0.025, "correct_lo": 0.2213433317104915, "correct_hi": 0.5049440300646263, "wrong_lo": 0.0044267174077949195, "wrong_hi": 0.1288165948800778, "verdict": "verify" }, { "model": "claude-opus-5.5", "track": "mcp", "job": "regulatory", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 40, "gradeable": 120, "correct": 68, "incorrect": 2, "abstained": 22, "no_answer": 28, "correct_pct": 0.5666666666666667, "wrong_pct": 0.016666666666666666, "correct_lo": 0.4140230650400791, "correct_hi": 0.7076269963932018, "wrong_lo": 0.002187647166978106, "wrong_hi": 0.1158494074417345, "verdict": "verify" }, { "model": "claude-opus-5.5", "track": "closed", "kind": "os-derived", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 13, "gradeable": 13, "correct": 0, "incorrect": 0, "abstained": 13, "no_answer": 0, "correct_pct": 0, "wrong_pct": 0, "correct_lo": 0, "correct_hi": 0.22810184305529166, "wrong_lo": 0, "wrong_hi": 0.22810184305529166, "verdict": "verify" }, { "model": "claude-opus-5.5", "track": "mcp", "kind": "os-derived", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 13, "gradeable": 39, "correct": 30, "incorrect": 3, "abstained": 2, "no_answer": 4, "correct_pct": 0.7692307692307693, "wrong_pct": 0.07692307692307693, "correct_lo": 0.49743126004873084, "correct_hi": 0.918206209075343, "wrong_lo": 0.013710066379696295, "wrong_hi": 0.3331453392824736, "verdict": "verify" }, { "model": "claude-sonnet-5.5", "track": "closed", "job": "baseline", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 25, "gradeable": 25, "correct": 25, "incorrect": 0, "abstained": 0, "no_answer": 0, "correct_pct": 1, "wrong_pct": 0, "correct_lo": 0.8668035060468213, "correct_hi": 1, "wrong_lo": 0, "wrong_hi": 0.13319649395317873, "verdict": "trust" }, { "model": "claude-sonnet-5.5", "track": "mcp", "job": "baseline", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 25, "gradeable": 75, "correct": 75, "incorrect": 0, "abstained": 0, "no_answer": 0, "correct_pct": 1, "wrong_pct": 0, "correct_lo": 0.8668035060468213, "correct_hi": 1, "wrong_lo": 0, "wrong_hi": 0.13319649395317873, "verdict": "trust" }, { "model": "claude-sonnet-5.5", "track": "closed", "job": "catalog", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 45, "gradeable": 45, "correct": 26, "incorrect": 11, "abstained": 8, "no_answer": 0, "correct_pct": 0.5777777777777777, "wrong_pct": 0.24444444444444444, "correct_lo": 0.433005327959445, "correct_hi": 0.7103151201831219, "wrong_lo": 0.14235335025509283, "wrong_hi": 0.3867366058479013, "verdict": "dont_use" }, { "model": "claude-sonnet-5.5", "track": "mcp", "job": "catalog", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 45, "gradeable": 135, "correct": 132, "incorrect": 3, "abstained": 0, "no_answer": 0, "correct_pct": 0.9777777777777777, "wrong_pct": 0.022222222222222223, "correct_lo": 0.8843305361128119, "correct_hi": 0.9960665024772425, "wrong_lo": 0.003933497522757505, "wrong_hi": 0.11566946388718813, "verdict": "trust" }, { "model": "claude-sonnet-5.5", "track": "closed", "job": "constellation", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 43, "gradeable": 43, "correct": 21, "incorrect": 5, "abstained": 17, "no_answer": 0, "correct_pct": 0.4883720930232558, "wrong_pct": 0.11627906976744186, "correct_lo": 0.34617209277756, "correct_hi": 0.6324793623392592, "wrong_lo": 0.050703529948620846, "wrong_hi": 0.24479448890641395, "verdict": "verify" }, { "model": "claude-sonnet-5.5", "track": "mcp", "job": "constellation", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 43, "gradeable": 129, "correct": 126, "incorrect": 2, "abstained": 1, "no_answer": 0, "correct_pct": 0.9767441860465116, "wrong_pct": 0.015503875968992248, "correct_lo": 0.8794073213878312, "correct_hi": 0.9958830188225801, "wrong_lo": 0.002034781229843595, "wrong_hi": 0.1084425153042913, "verdict": "trust" }, { "model": "claude-sonnet-5.5", "track": "closed", "job": "debris", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 42, "gradeable": 42, "correct": 19, "incorrect": 7, "abstained": 16, "no_answer": 0, "correct_pct": 0.4523809523809524, "wrong_pct": 0.16666666666666666, "correct_lo": 0.3122316381863852, "correct_hi": 0.6005113725287732, "wrong_lo": 0.08315811295836502, "wrong_hi": 0.30604296204774295, "verdict": "verify" }, { "model": "claude-sonnet-5.5", "track": "mcp", "job": "debris", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 42, "gradeable": 126, "correct": 117, "incorrect": 5, "abstained": 3, "no_answer": 1, "correct_pct": 0.9285714285714286, "wrong_pct": 0.03968253968253968, "correct_lo": 0.8099028671147483, "correct_hi": 0.9754100364488272, "wrong_lo": 0.009835992727494747, "wrong_hi": 0.14667977751903535, "verdict": "trust" }, { "model": "claude-sonnet-5.5", "track": "closed", "job": "ground", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 22, "gradeable": 22, "correct": 4, "incorrect": 6, "abstained": 12, "no_answer": 0, "correct_pct": 0.18181818181818182, "wrong_pct": 0.2727272727272727, "correct_lo": 0.07306766024105121, "correct_hi": 0.38517021976637866, "wrong_lo": 0.13150582015814438, "wrong_hi": 0.4815212369900198, "verdict": "dont_use" }, { "model": "claude-sonnet-5.5", "track": "mcp", "job": "ground", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 22, "gradeable": 66, "correct": 49, "incorrect": 11, "abstained": 3, "no_answer": 3, "correct_pct": 0.7424242424242424, "wrong_pct": 0.16666666666666666, "correct_lo": 0.5339703167166809, "correct_hi": 0.8788008223252772, "wrong_lo": 0.06422420536802462, "wrong_hi": 0.36821547844928293, "verdict": "verify" }, { "model": "claude-sonnet-5.5", "track": "closed", "job": "market", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 41, "gradeable": 41, "correct": 17, "incorrect": 11, "abstained": 13, "no_answer": 0, "correct_pct": 0.4146341463414634, "wrong_pct": 0.2682926829268293, "correct_lo": 0.2775633082533015, "correct_hi": 0.5663316464316339, "wrong_lo": 0.15694805281336685, "wrong_hi": 0.41933825276002934, "verdict": "dont_use" }, { "model": "claude-sonnet-5.5", "track": "mcp", "job": "market", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 41, "gradeable": 123, "correct": 102, "incorrect": 3, "abstained": 5, "no_answer": 13, "correct_pct": 0.8292682926829268, "wrong_pct": 0.024390243902439025, "correct_lo": 0.6873709004537024, "correct_hi": 0.914748560047261, "wrong_lo": 0.00431842027071648, "wrong_hi": 0.12595347011678087, "verdict": "trust" }, { "model": "claude-sonnet-5.5", "track": "closed", "job": "regulatory", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 40, "gradeable": 40, "correct": 11, "incorrect": 4, "abstained": 25, "no_answer": 0, "correct_pct": 0.275, "wrong_pct": 0.1, "correct_lo": 0.1610785343314309, "correct_hi": 0.4283525083312457, "wrong_lo": 0.03957888759557489, "wrong_hi": 0.23052074380473903, "verdict": "verify" }, { "model": "claude-sonnet-5.5", "track": "mcp", "job": "regulatory", "kind": "public-verifiable", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 40, "gradeable": 120, "correct": 56, "incorrect": 17, "abstained": 34, "no_answer": 13, "correct_pct": 0.4666666666666667, "wrong_pct": 0.14166666666666666, "correct_lo": 0.32188066221605804, "correct_hi": 0.6172943070673015, "wrong_lo": 0.06517299126498999, "wrong_hi": 0.28095792853112456, "verdict": "verify" }, { "model": "claude-sonnet-5.5", "track": "closed", "kind": "os-derived", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 1, "questions": 13, "gradeable": 13, "correct": 0, "incorrect": 1, "abstained": 12, "no_answer": 0, "correct_pct": 0, "wrong_pct": 0.07692307692307693, "correct_lo": 0, "correct_hi": 0.22810184305529166, "wrong_lo": 0.013710066379696295, "wrong_hi": 0.3331453392824736, "verdict": "verify" }, { "model": "claude-sonnet-5.5", "track": "mcp", "kind": "os-derived", "provider": "claude-code", "harness": "2.1.296 (Claude Code)", "runs": 3, "questions": 13, "gradeable": 39, "correct": 30, "incorrect": 4, "abstained": 1, "no_answer": 4, "correct_pct": 0.7692307692307693, "wrong_pct": 0.10256410256410256, "correct_lo": 0.49743126004873084, "correct_hi": 0.918206209075343, "wrong_lo": 0.022298775323178394, "wrong_hi": 0.3641411512079509, "verdict": "verify" }, { "model": "deepseek-r1-14b", "track": "closed", "job": "baseline", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 24, "gradeable": 24, "correct": 14, "incorrect": 10, "abstained": 0, "no_answer": 0, "correct_pct": 0.5833333333333334, "wrong_pct": 0.4166666666666667, "correct_lo": 0.3883433764938864, "correct_hi": 0.7553265275346177, "wrong_lo": 0.24467347246538232, "wrong_hi": 0.6116566235061137, "verdict": "dont_use" }, { "model": "deepseek-r1-14b", "track": "closed", "job": "catalog", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 45, "gradeable": 45, "correct": 1, "incorrect": 34, "abstained": 10, "no_answer": 0, "correct_pct": 0.022222222222222223, "wrong_pct": 0.7555555555555555, "correct_lo": 0.003933497522757505, "correct_hi": 0.11566946388718813, "wrong_lo": 0.6132633941520986, "wrong_hi": 0.8576466497449071, "verdict": "dont_use" }, { "model": "deepseek-r1-14b", "track": "closed", "job": "constellation", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 43, "gradeable": 43, "correct": 3, "incorrect": 29, "abstained": 11, "no_answer": 0, "correct_pct": 0.06976744186046512, "wrong_pct": 0.6744186046511628, "correct_lo": 0.024011153791962503, "correct_hi": 0.18609268553034927, "wrong_lo": 0.5251593250655096, "wrong_hi": 0.7950688481822019, "verdict": "dont_use" }, { "model": "deepseek-r1-14b", "track": "closed", "job": "debris", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 42, "gradeable": 42, "correct": 5, "incorrect": 21, "abstained": 16, "no_answer": 0, "correct_pct": 0.11904761904761904, "wrong_pct": 0.5, "correct_lo": 0.05193726765019084, "correct_hi": 0.2500068180710754, "wrong_lo": 0.3552574591652803, "wrong_hi": 0.6447425408347197, "verdict": "dont_use" }, { "model": "deepseek-r1-14b", "track": "closed", "job": "ground", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 22, "gradeable": 22, "correct": 3, "incorrect": 8, "abstained": 11, "no_answer": 0, "correct_pct": 0.13636363636363635, "wrong_pct": 0.36363636363636365, "correct_lo": 0.04748917264236269, "correct_hi": 0.3333541187947, "wrong_lo": 0.19732972772607899, "wrong_hi": 0.5704865065628195, "verdict": "dont_use" }, { "model": "deepseek-r1-14b", "track": "closed", "job": "market", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 41, "gradeable": 41, "correct": 1, "incorrect": 18, "abstained": 22, "no_answer": 0, "correct_pct": 0.024390243902439025, "wrong_pct": 0.43902439024390244, "correct_lo": 0.00431842027071648, "correct_hi": 0.12595347011678087, "wrong_lo": 0.2988990477210983, "wrong_hi": 0.5895973484824271, "verdict": "dont_use" }, { "model": "deepseek-r1-14b", "track": "closed", "job": "regulatory", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 40, "gradeable": 40, "correct": 3, "incorrect": 20, "abstained": 17, "no_answer": 0, "correct_pct": 0.075, "wrong_pct": 0.5, "correct_lo": 0.025835556771858226, "correct_hi": 0.19864530159097524, "wrong_lo": 0.35199278797099753, "wrong_hi": 0.6480072120290025, "verdict": "dont_use" }, { "model": "deepseek-r1-14b", "track": "closed", "kind": "os-derived", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 13, "gradeable": 13, "correct": 0, "incorrect": 4, "abstained": 9, "no_answer": 0, "correct_pct": 0, "wrong_pct": 0.3076923076923077, "correct_lo": 0, "correct_hi": 0.22810184305529166, "wrong_lo": 0.12680497161618087, "wrong_hi": 0.5763111218666236, "verdict": "dont_use" }, { "model": "gemma4-12b", "track": "closed", "job": "baseline", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 13, "gradeable": 13, "correct": 12, "incorrect": 1, "abstained": 0, "no_answer": 0, "correct_pct": 0.9230769230769231, "wrong_pct": 0.07692307692307693, "correct_lo": 0.6668546607175265, "correct_hi": 0.9862899336203037, "wrong_lo": 0.013710066379696295, "wrong_hi": 0.3331453392824736, "verdict": "verify" }, { "model": "gemma4-12b", "track": "mcp", "job": "baseline", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 23, "gradeable": 64, "correct": 57, "incorrect": 2, "abstained": 5, "no_answer": 0, "correct_pct": 0.890625, "wrong_pct": 0.03125, "correct_lo": 0.7040764768585807, "correct_hi": 0.9653601439091829, "wrong_lo": 0.00435094393543034, "wrong_hi": 0.1923251111432535, "verdict": "trust" }, { "model": "gemma4-12b", "track": "closed", "job": "catalog", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 38, "gradeable": 38, "correct": 0, "incorrect": 26, "abstained": 12, "no_answer": 0, "correct_pct": 0, "wrong_pct": 0.6842105263157895, "correct_lo": 0, "correct_hi": 0.09181293258384, "wrong_lo": 0.5254394356005149, "wrong_hi": 0.8091557997633335, "verdict": "dont_use" }, { "model": "gemma4-12b", "track": "mcp", "job": "catalog", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 45, "gradeable": 130, "correct": 101, "incorrect": 19, "abstained": 10, "no_answer": 0, "correct_pct": 0.7769230769230769, "wrong_pct": 0.14615384615384616, "correct_lo": 0.6363720610905004, "correct_hi": 0.8739117322962218, "wrong_lo": 0.07107720955275645, "wrong_hi": 0.2768934988975432, "verdict": "verify" }, { "model": "gemma4-12b", "track": "closed", "job": "constellation", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 33, "gradeable": 33, "correct": 2, "incorrect": 19, "abstained": 12, "no_answer": 0, "correct_pct": 0.06060606060606061, "wrong_pct": 0.5757575757575758, "correct_lo": 0.01678053042907096, "correct_hi": 0.19606581717798194, "wrong_lo": 0.4080697369146003, "wrong_hi": 0.7276464100497011, "verdict": "dont_use" }, { "model": "gemma4-12b", "track": "mcp", "job": "constellation", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 43, "gradeable": 128, "correct": 54, "incorrect": 52, "abstained": 21, "no_answer": 1, "correct_pct": 0.421875, "wrong_pct": 0.40625, "correct_lo": 0.28670670404341114, "correct_hi": 0.5698577600227182, "wrong_lo": 0.27307901537662815, "wrong_hi": 0.554798341502727, "verdict": "dont_use" }, { "model": "gemma4-12b", "track": "closed", "job": "debris", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 41, "gradeable": 41, "correct": 4, "incorrect": 14, "abstained": 23, "no_answer": 0, "correct_pct": 0.0975609756097561, "wrong_pct": 0.34146341463414637, "correct_lo": 0.03859645975296143, "correct_hi": 0.22547975519030555, "wrong_lo": 0.2155861736664346, "wrong_hi": 0.4945044564627312, "verdict": "dont_use" }, { "model": "gemma4-12b", "track": "mcp", "job": "debris", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 42, "gradeable": 125, "correct": 65, "incorrect": 22, "abstained": 38, "no_answer": 0, "correct_pct": 0.52, "wrong_pct": 0.176, "correct_lo": 0.3736875561295388, "correct_hi": 0.6629603793700949, "wrong_lo": 0.08961555603586711, "wrong_hi": 0.3166878888700699, "verdict": "verify" }, { "model": "gemma4-12b", "track": "closed", "job": "ground", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 22, "gradeable": 22, "correct": 0, "incorrect": 10, "abstained": 12, "no_answer": 0, "correct_pct": 0, "wrong_pct": 0.45454545454545453, "correct_lo": 0, "correct_hi": 0.14865952572596125, "wrong_lo": 0.26920014432004846, "wrong_hi": 0.6534052671095845, "verdict": "dont_use" }, { "model": "gemma4-12b", "track": "mcp", "job": "ground", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 22, "gradeable": 64, "correct": 12, "incorrect": 37, "abstained": 14, "no_answer": 1, "correct_pct": 0.1875, "wrong_pct": 0.578125, "correct_lo": 0.0764584223912903, "correct_hi": 0.39145378118743546, "wrong_lo": 0.3757429969450212, "wrong_hi": 0.7572789521602974, "verdict": "dont_use" }, { "model": "gemma4-12b", "track": "closed", "job": "market", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 36, "gradeable": 36, "correct": 3, "incorrect": 11, "abstained": 22, "no_answer": 0, "correct_pct": 0.08333333333333333, "wrong_pct": 0.3055555555555556, "correct_lo": 0.0287480892712918, "correct_hi": 0.21827010276918848, "wrong_lo": 0.18004270282805354, "wrong_hi": 0.46856578679083716, "verdict": "dont_use" }, { "model": "gemma4-12b", "track": "mcp", "job": "market", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 41, "gradeable": 120, "correct": 56, "incorrect": 43, "abstained": 21, "no_answer": 0, "correct_pct": 0.4666666666666667, "wrong_pct": 0.35833333333333334, "correct_lo": 0.3234724330366314, "correct_hi": 0.6155722635546291, "wrong_lo": 0.22959573810529016, "wrong_hi": 0.511344222407567, "verdict": "dont_use" }, { "model": "gemma4-12b", "track": "closed", "job": "regulatory", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 36, "gradeable": 36, "correct": 4, "incorrect": 16, "abstained": 16, "no_answer": 0, "correct_pct": 0.1111111111111111, "wrong_pct": 0.4444444444444444, "correct_lo": 0.04406568908741833, "correct_hi": 0.25315129015036325, "wrong_lo": 0.2954102987654314, "wrong_hi": 0.6041921268399659, "verdict": "dont_use" }, { "model": "gemma4-12b", "track": "mcp", "job": "regulatory", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 40, "gradeable": 120, "correct": 16, "incorrect": 31, "abstained": 73, "no_answer": 0, "correct_pct": 0.13333333333333333, "wrong_pct": 0.25833333333333336, "correct_lo": 0.05983196228036035, "correct_hi": 0.27109269983659406, "wrong_lo": 0.14821936174138453, "wrong_hi": 0.4107991655629719, "verdict": "dont_use" }, { "model": "gemma4-12b", "track": "closed", "kind": "os-derived", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 13, "gradeable": 13, "correct": 0, "incorrect": 5, "abstained": 8, "no_answer": 0, "correct_pct": 0, "wrong_pct": 0.38461538461538464, "correct_lo": 0, "correct_hi": 0.22810184305529166, "wrong_lo": 0.17709446581236818, "wrong_hi": 0.6447751902773146, "verdict": "dont_use" }, { "model": "gemma4-12b", "track": "mcp", "kind": "os-derived", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 13, "gradeable": 36, "correct": 27, "incorrect": 5, "abstained": 4, "no_answer": 0, "correct_pct": 0.75, "wrong_pct": 0.1388888888888889, "correct_lo": 0.4784494853422473, "correct_hi": 0.907499593130107, "wrong_lo": 0.03669049170196942, "wrong_hi": 0.40582750606018575, "verdict": "verify" }, { "model": "gpt-oss-20b", "track": "closed", "job": "baseline", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 24, "gradeable": 24, "correct": 18, "incorrect": 4, "abstained": 0, "no_answer": 2, "correct_pct": 0.75, "wrong_pct": 0.16666666666666666, "correct_lo": 0.5510017468789357, "correct_hi": 0.8800079652065766, "wrong_lo": 0.06678568000728921, "wrong_hi": 0.35853470387869435, "verdict": "verify" }, { "model": "gpt-oss-20b", "track": "mcp", "job": "baseline", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 25, "gradeable": 75, "correct": 64, "incorrect": 5, "abstained": 0, "no_answer": 6, "correct_pct": 0.8533333333333334, "wrong_pct": 0.06666666666666667, "correct_lo": 0.6688471946120905, "correct_hi": 0.9436939496609965, "wrong_lo": 0.016592731775468883, "wrong_hi": 0.23217756298395267, "verdict": "verify" }, { "model": "gpt-oss-20b", "track": "closed", "job": "catalog", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 45, "gradeable": 45, "correct": 3, "incorrect": 28, "abstained": 14, "no_answer": 0, "correct_pct": 0.06666666666666667, "wrong_pct": 0.6222222222222222, "correct_lo": 0.0229316164621471, "correct_hi": 0.17856874388640825, "wrong_lo": 0.4762967000646754, "wrong_hi": 0.7489211470165014, "verdict": "dont_use" }, { "model": "gpt-oss-20b", "track": "mcp", "job": "catalog", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 45, "gradeable": 135, "correct": 102, "incorrect": 14, "abstained": 2, "no_answer": 17, "correct_pct": 0.7555555555555555, "wrong_pct": 0.1037037037037037, "correct_lo": 0.6132633941520986, "correct_hi": 0.8576466497449071, "wrong_lo": 0.04386614242327462, "wrong_hi": 0.22588205037412212, "verdict": "verify" }, { "model": "gpt-oss-20b", "track": "closed", "job": "constellation", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 43, "gradeable": 43, "correct": 2, "incorrect": 22, "abstained": 19, "no_answer": 0, "correct_pct": 0.046511627906976744, "wrong_pct": 0.5116279069767442, "correct_lo": 0.012848990317319689, "correct_hi": 0.15455775923863058, "wrong_lo": 0.36752063766074083, "wrong_hi": 0.6538279072224399, "verdict": "dont_use" }, { "model": "gpt-oss-20b", "track": "mcp", "job": "constellation", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 43, "gradeable": 129, "correct": 38, "incorrect": 47, "abstained": 4, "no_answer": 40, "correct_pct": 0.29457364341085274, "wrong_pct": 0.3643410852713178, "correct_lo": 0.17979272202216878, "correct_hi": 0.44304965170830446, "wrong_lo": 0.2372003425387799, "wrong_hi": 0.5137333004907779, "verdict": "dont_use" }, { "model": "gpt-oss-20b", "track": "closed", "job": "debris", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 42, "gradeable": 42, "correct": 5, "incorrect": 27, "abstained": 10, "no_answer": 0, "correct_pct": 0.11904761904761904, "wrong_pct": 0.6428571428571429, "correct_lo": 0.05193726765019084, "correct_hi": 0.2500068180710754, "wrong_lo": 0.4916608928707938, "wrong_hi": 0.7701100749837315, "verdict": "dont_use" }, { "model": "gpt-oss-20b", "track": "mcp", "job": "debris", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 42, "gradeable": 126, "correct": 56, "incorrect": 24, "abstained": 9, "no_answer": 37, "correct_pct": 0.4444444444444444, "wrong_pct": 0.19047619047619047, "correct_lo": 0.3051784747834041, "correct_hi": 0.5930217043842806, "wrong_lo": 0.09981889609856438, "wrong_hi": 0.3330106735499645, "verdict": "verify" }, { "model": "gpt-oss-20b", "track": "closed", "job": "ground", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 22, "gradeable": 22, "correct": 2, "incorrect": 10, "abstained": 10, "no_answer": 0, "correct_pct": 0.09090909090909091, "wrong_pct": 0.45454545454545453, "correct_lo": 0.025294896282837764, "correct_hi": 0.27815380658385785, "wrong_lo": 0.26920014432004846, "wrong_hi": 0.6534052671095845, "verdict": "dont_use" }, { "model": "gpt-oss-20b", "track": "mcp", "job": "ground", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 22, "gradeable": 66, "correct": 15, "incorrect": 12, "abstained": 11, "no_answer": 28, "correct_pct": 0.22727272727272727, "wrong_pct": 0.18181818181818182, "correct_lo": 0.10122890794820358, "correct_hi": 0.4344035606295935, "wrong_lo": 0.07306766024105121, "wrong_hi": 0.38517021976637866, "verdict": "verify" }, { "model": "gpt-oss-20b", "track": "closed", "job": "market", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 41, "gradeable": 41, "correct": 3, "incorrect": 24, "abstained": 14, "no_answer": 0, "correct_pct": 0.07317073170731707, "wrong_pct": 0.5853658536585366, "correct_lo": 0.02519737325256413, "correct_hi": 0.19427740017211295, "wrong_lo": 0.433668353568366, "wrong_hi": 0.7224366917466984, "verdict": "dont_use" }, { "model": "gpt-oss-20b", "track": "mcp", "job": "market", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 41, "gradeable": 123, "correct": 57, "incorrect": 26, "abstained": 6, "no_answer": 34, "correct_pct": 0.4634146341463415, "wrong_pct": 0.21138211382113822, "correct_lo": 0.32055999118185885, "correct_hi": 0.6125378465402564, "wrong_lo": 0.11407283240921819, "wrong_hi": 0.3581434429541349, "verdict": "dont_use" }, { "model": "gpt-oss-20b", "track": "closed", "job": "regulatory", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 40, "gradeable": 40, "correct": 3, "incorrect": 24, "abstained": 13, "no_answer": 0, "correct_pct": 0.075, "wrong_pct": 0.6, "correct_lo": 0.025835556771858226, "correct_hi": 0.19864530159097524, "wrong_lo": 0.445956198273935, "wrong_hi": 0.7365188938759863, "verdict": "dont_use" }, { "model": "gpt-oss-20b", "track": "mcp", "job": "regulatory", "kind": "public-verifiable", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 40, "gradeable": 120, "correct": 16, "incorrect": 30, "abstained": 10, "no_answer": 64, "correct_pct": 0.13333333333333333, "wrong_pct": 0.25, "correct_lo": 0.05983196228036035, "correct_hi": 0.27109269983659406, "wrong_lo": 0.14186967895549518, "wrong_hi": 0.40194259066970106, "verdict": "dont_use" }, { "model": "gpt-oss-20b", "track": "closed", "kind": "os-derived", "provider": "ollama", "harness": "spacebench-runner", "runs": 1, "questions": 13, "gradeable": 13, "correct": 0, "incorrect": 5, "abstained": 8, "no_answer": 0, "correct_pct": 0, "wrong_pct": 0.38461538461538464, "correct_lo": 0, "correct_hi": 0.22810184305529166, "wrong_lo": 0.17709446581236818, "wrong_hi": 0.6447751902773146, "verdict": "dont_use" }, { "model": "gpt-oss-20b", "track": "mcp", "kind": "os-derived", "provider": "ollama", "harness": "spacebench-runner", "runs": 3, "questions": 13, "gradeable": 39, "correct": 26, "incorrect": 11, "abstained": 2, "no_answer": 0, "correct_pct": 0.6666666666666666, "wrong_pct": 0.28205128205128205, "correct_lo": 0.4003196931375318, "correct_hi": 0.8569796925107044, "wrong_lo": 0.1111716335484369, "wrong_hi": 0.5523599390654081, "verdict": "dont_use" }, { "model": "grok-4.7", "track": "closed", "job": "baseline", "kind": "public-verifiable", "provider": "xai", "harness": "spacebench-runner", "runs": 1, "questions": 25, "gradeable": 25, "correct": 24, "incorrect": 0, "abstained": 1, "no_answer": 0, "correct_pct": 0.96, "wrong_pct": 0, "correct_lo": 0.8045552759497747, "correct_hi": 0.9929039496133008, "wrong_lo": 0, "wrong_hi": 0.13319649395317873, "verdict": "trust" }, { "model": "grok-4.7", "track": "mcp", "job": "baseline", "kind": "public-verifiable", "provider": "xai", "harness": "spacebench-runner", "runs": 2, "questions": 25, "gradeable": 50, "correct": 49, "incorrect": 0, "abstained": 1, "no_answer": 0, "correct_pct": 0.98, "wrong_pct": 0, "correct_lo": 0.8342228568014084, "correct_hi": 0.9979085090035399, "wrong_lo": 0, "wrong_hi": 0.13319649395317873, "verdict": "trust" }, { "model": "grok-4.7", "track": "closed", "job": "catalog", "kind": "public-verifiable", "provider": "xai", "harness": "spacebench-runner", "runs": 1, "questions": 45, "gradeable": 45, "correct": 21, "incorrect": 0, "abstained": 24, "no_answer": 0, "correct_pct": 0.4666666666666667, "wrong_pct": 0, "correct_lo": 0.3293490394201473, "correct_hi": 0.6092279113758954, "wrong_lo": 6.938893903907228e-18, "wrong_hi": 0.07865426194064076, "verdict": "verify" }, { "model": "grok-4.7", "track": "mcp", "job": "catalog", "kind": "public-verifiable", "provider": "xai", "harness": "spacebench-runner", "runs": 2, "questions": 45, "gradeable": 90, "correct": 82, "incorrect": 1, "abstained": 7, "no_answer": 0, "correct_pct": 0.9111111111111111, "wrong_pct": 0.011111111111111112, "correct_lo": 0.792661488316343, "correct_hi": 0.9648894518657969, "wrong_lo": 0.0011610618786614471, "wrong_hi": 0.09796754979663178, "verdict": "trust" }, { "model": "grok-4.7", "track": "closed", "job": "constellation", "kind": "public-verifiable", "provider": "xai", "harness": "spacebench-runner", "runs": 1, "questions": 43, "gradeable": 43, "correct": 8, "incorrect": 0, "abstained": 35, "no_answer": 0, "correct_pct": 0.18604651162790697, "wrong_pct": 0, "correct_lo": 0.0974163558010598, "correct_hi": 0.3261729323530596, "wrong_lo": 0, "wrong_hi": 0.08201257002322722, "verdict": "verify" }, { "model": "grok-4.7", "track": "mcp", "job": "constellation", "kind": "public-verifiable", "provider": "xai", "harness": "spacebench-runner", "runs": 2, "questions": 43, "gradeable": 86, "correct": 82, "incorrect": 4, "abstained": 0, "no_answer": 0, "correct_pct": 0.9534883720930233, "wrong_pct": 0.046511627906976744, "correct_lo": 0.8454422407613695, "correct_hi": 0.9871510096826803, "wrong_lo": 0.012848990317319689, "wrong_hi": 0.15455775923863058, "verdict": "trust" }, { "model": "grok-4.7", "track": "closed", "job": "debris", "kind": "public-verifiable", "provider": "xai", "harness": "spacebench-runner", "runs": 1, "questions": 42, "gradeable": 42, "correct": 5, "incorrect": 2, "abstained": 35, "no_answer": 0, "correct_pct": 0.11904761904761904, "wrong_pct": 0.047619047619047616, "correct_lo": 0.05193726765019084, "correct_hi": 0.2500068180710754, "wrong_lo": 0.013157248639206379, "wrong_hi": 0.1579013531547973, "verdict": "verify" }, { "model": "grok-4.7", "track": "mcp", "job": "debris", "kind": "public-verifiable", "provider": "xai", "harness": "spacebench-runner", "runs": 2, "questions": 42, "gradeable": 84, "correct": 71, "incorrect": 3, "abstained": 10, "no_answer": 0, "correct_pct": 0.8452380952380952, "wrong_pct": 0.03571428571428571, "correct_lo": 0.7076825728932761, "correct_hi": 0.9249305994218264, "wrong_lo": 0.008290820071836139, "wrong_hi": 0.14095353440095715, "verdict": "trust" }, { "model": "grok-4.7", "track": "closed", "job": "ground", "kind": "public-verifiable", "provider": "xai", "harness": "spacebench-runner", "runs": 1, "questions": 22, "gradeable": 22, "correct": 3, "incorrect": 1, "abstained": 18, "no_answer": 0, "correct_pct": 0.13636363636363635, "wrong_pct": 0.045454545454545456, "correct_lo": 0.04748917264236269, "correct_hi": 0.3333541187947, "wrong_lo": 0.008069219301147859, "wrong_hi": 0.21798489499518053, "verdict": "verify" }, { "model": "grok-4.7", "track": "mcp", "job": "ground", "kind": "public-verifiable", "provider": "xai", "harness": "spacebench-runner", "runs": 2, "questions": 22, "gradeable": 44, "correct": 25, "incorrect": 5, "abstained": 14, "no_answer": 0, "correct_pct": 0.5681818181818182, "wrong_pct": 0.11363636363636363, "correct_lo": 0.36679601352424396, "correct_hi": 0.7492958693313069, "wrong_lo": 0.03589747812594399, "wrong_hi": 0.3062485190259352, "verdict": "verify" }, { "model": "grok-4.7", "track": "closed", "job": "market", "kind": "public-verifiable", "provider": "xai", "harness": "spacebench-runner", "runs": 1, "questions": 41, "gradeable": 41, "correct": 8, "incorrect": 2, "abstained": 31, "no_answer": 0, "correct_pct": 0.1951219512195122, "wrong_pct": 0.04878048780487805, "correct_lo": 0.10234315749205929, "correct_hi": 0.34013882352556724, "wrong_lo": 0.013480662282573552, "wrong_hi": 0.16139266962351367, "verdict": "verify" }, { "model": "grok-4.7", "track": "mcp", "job": "market", "kind": "public-verifiable", "provider": "xai", "harness": "spacebench-runner", "runs": 2, "questions": 41, "gradeable": 82, "correct": 64, "incorrect": 2, "abstained": 16, "no_answer": 0, "correct_pct": 0.7804878048780488, "wrong_pct": 0.024390243902439025, "correct_lo": 0.6329469790176472, "correct_hi": 0.8799695984461366, "wrong_lo": 0.00431842027071648, "wrong_hi": 0.12595347011678087, "verdict": "verify" }, { "model": "grok-4.7", "track": "closed", "job": "regulatory", "kind": "public-verifiable", "provider": "xai", "harness": "spacebench-runner", "runs": 1, "questions": 40, "gradeable": 40, "correct": 11, "incorrect": 0, "abstained": 29, "no_answer": 0, "correct_pct": 0.275, "wrong_pct": 0, "correct_lo": 0.1610785343314309, "correct_hi": 0.4283525083312457, "wrong_lo": 0, "wrong_hi": 0.08762453925039233, "verdict": "verify" }, { "model": "grok-4.7", "track": "mcp", "job": "regulatory", "kind": "public-verifiable", "provider": "xai", "harness": "spacebench-runner", "runs": 2, "questions": 40, "gradeable": 80, "correct": 35, "incorrect": 3, "abstained": 42, "no_answer": 0, "correct_pct": 0.4375, "wrong_pct": 0.0375, "correct_lo": 0.29602809418250353, "correct_hi": 0.5899249732237956, "wrong_lo": 0.008707641902894095, "wrong_hi": 0.1473450569037188, "verdict": "verify" }, { "model": "grok-4.7", "track": "closed", "kind": "os-derived", "provider": "xai", "harness": "spacebench-runner", "runs": 1, "questions": 13, "gradeable": 13, "correct": 0, "incorrect": 0, "abstained": 13, "no_answer": 0, "correct_pct": 0, "wrong_pct": 0, "correct_lo": 0, "correct_hi": 0.22810184305529166, "wrong_lo": 0, "wrong_hi": 0.22810184305529166, "verdict": "verify" }, { "model": "grok-4.7", "track": "mcp", "kind": "os-derived", "provider": "xai", "harness": "spacebench-runner", "runs": 2, "questions": 13, "gradeable": 26, "correct": 19, "incorrect": 2, "abstained": 5, "no_answer": 0, "correct_pct": 0.7307692307692307, "wrong_pct": 0.07692307692307693, "correct_lo": 0.4598441149709075, "correct_hi": 0.896416572849727, "wrong_lo": 0.013710066379696295, "wrong_hi": 0.3331453392824736, "verdict": "verify" } ] }