{
"task_count": 100,
"current_native_runs": 3300,
"current_uncapped_search_runs": 1350,
"retained_run_count": 4650,
"tasks": 100,
"runs": 4650,
"native_runs": 3300,
"ablation_runs": 1350,
"current_evaluation_runs": 4650,
"native_models": [
{
"id": "google/gemini-4-argon",
"name": "Gemini 4 Argon",
"lab": "Google",
"strict": 72.3,
"strict_ci": [
64.0,
80.0
],
"mean": 94.1,
"mean_ci": [
92.0,
95.9
],
"all3": 64,
"counts": [
20,
7,
9,
64
],
"hat3": 64.0,
"hat3_ci": [
54.0,
73.0
]
},
{
"id": "gpt-6-astra",
"name": "GPT-6 Astra",
"lab": "OpenAI",
"strict": 71.3,
"strict_ci": [
63.3,
78.7
],
"mean": 92.7,
"mean_ci": [
90.3,
94.8
],
"all3": 60,
"counts": [
19,
8,
13,
60
],
"hat3": 60.0,
"hat3_ci": [
50.0,
69.0
]
},
{
"id": "anthropic/claude-opus-5.5",
"name": "Opus 5.5",
"lab": "Anthropic",
"strict": 70.3,
"strict_ci": [
62.3,
77.7
],
"mean": 94.3,
"mean_ci": [
92.3,
95.9
],
"all3": 57,
"counts": [
17,
12,
14,
57
],
"hat3": 57.0,
"hat3_ci": [
47.0,
67.0
]
},
{
"id": "gpt-6.1-sol",
"name": "GPT-6.1 Sol",
"lab": "OpenAI",
"strict": 68.0,
"strict_ci": [
59.7,
76.0
],
"mean": 91.2,
"mean_ci": [
88.4,
93.8
],
"all3": 59,
"counts": [
21,
13,
7,
59
],
"hat3": 59.0,
"hat3_ci": [
49.0,
68.0
]
},
{
"id": "gpt-5.6-sol",
"name": "GPT-5.6 Sol",
"lab": "OpenAI",
"strict": 59.0,
"strict_ci": [
51.0,
66.7
],
"mean": 88.1,
"mean_ci": [
84.6,
91.1
],
"all3": 38,
"counts": [
22,
17,
23,
38
],
"hat3": 38.0,
"hat3_ci": [
29.0,
48.0
]
},
{
"id": "anthropic/claude-opus-5",
"name": "Opus 5",
"lab": "Anthropic",
"strict": 54.3,
"strict_ci": [
46.3,
62.3
],
"mean": 88.2,
"mean_ci": [
85.3,
90.9
],
"all3": 34,
"counts": [
29,
13,
24,
34
],
"hat3": 34.0,
"hat3_ci": [
25.0,
44.0
]
},
{
"id": "x-ai/grok-4.6",
"name": "Grok 4.6",
"lab": "xAI",
"strict": 53.3,
"strict_ci": [
45.0,
61.3
],
"mean": 87.3,
"mean_ci": [
84.2,
90.2
],
"all3": 35,
"counts": [
31,
13,
21,
35
],
"hat3": 35.0,
"hat3_ci": [
26.0,
44.0
]
},
{
"id": "google/gemini-3.8-flash",
"name": "Gemini 3.8 Flash",
"lab": "Google",
"strict": 45.0,
"strict_ci": [
37.0,
53.3
],
"mean": 83.0,
"mean_ci": [
79.1,
86.6
],
"all3": 29,
"counts": [
37,
20,
14,
29
],
"hat3": 29.0,
"hat3_ci": [
20.0,
38.0
]
},
{
"id": "thinkingmachines/Inkling-2-Small:peft:262144",
"name": "Inkling 2 Small",
"lab": "Thinking Machines",
"strict": 44.0,
"mean": 82.8,
"all3": 23,
"hat3": 23.0,
"counts": [
35,
21,
21,
23
]
},
{
"id": "moonshotai/kimi-k3",
"name": "Kimi K3",
"lab": "Moonshot AI",
"strict": 32.3,
"strict_ci": [
25.0,
39.7
],
"mean": 76.6,
"mean_ci": [
72.4,
80.5
],
"all3": 15,
"counts": [
49,
20,
16,
15
],
"hat3": 15.0,
"hat3_ci": [
8.0,
22.0
]
},
{
"id": "anthropic/claude-sonnet-5",
"name": "Sonnet 5",
"lab": "Anthropic",
"strict": 27.0,
"strict_ci": [
21.0,
33.3
],
"mean": 72.0,
"mean_ci": [
67.5,
76.3
],
"all3": 7,
"counts": [
51,
24,
18,
7
],
"hat3": 7.0,
"hat3_ci": [
2.0,
12.0
]
}
],
"efficiency": [
{
"id": "anthropic/claude-opus-5",
"name": "Opus 5",
"lab": "Anthropic",
"strict": 54.3,
"spend": 3.3,
"cost": 1.43,
"minutes": 30.1,
"tokens": 74856
},
{
"id": "anthropic/claude-opus-5.5",
"name": "Opus 5.5",
"lab": "Anthropic",
"strict": 70.3,
"spend": 1.31,
"cost": 0.85,
"minutes": 15.5,
"tokens": 42376
},
{
"id": "anthropic/claude-sonnet-5",
"name": "Sonnet 5",
"lab": "Anthropic",
"strict": 27.0,
"spend": 3.04,
"cost": 0.54,
"minutes": 17.7,
"tokens": 60656
},
{
"id": "google/gemini-3.8-flash",
"name": "Gemini 3.8 Flash",
"lab": "Google",
"strict": 45.0,
"spend": 2.49,
"cost": 0.85,
"minutes": 14.2,
"tokens": 55176,
"cost_estimated": true
},
{
"id": "gpt-5.6-sol",
"name": "GPT-5.6 Sol",
"lab": "OpenAI",
"strict": 59.0,
"spend": 2.68,
"cost": 1.21,
"minutes": 12.2,
"tokens": 30446
},
{
"id": "gpt-6-astra",
"name": "GPT-6 Astra",
"lab": "OpenAI",
"strict": 71.3,
"spend": 5.22,
"cost": 2.51,
"minutes": 10.0,
"tokens": 21260
},
{
"id": "gpt-6.1-sol",
"name": "GPT-6.1 Sol",
"lab": "OpenAI",
"strict": 68.0,
"spend": 0.98,
"cost": 0.48,
"minutes": 7.6,
"tokens": 22432
},
{
"id": "moonshotai/kimi-k3",
"name": "Kimi K3",
"lab": "Moonshot AI",
"strict": 32.3,
"spend": 0.85,
"cost": 0.2,
"minutes": 14.5,
"tokens": 14840,
"cost_estimated": true
},
{
"id": "x-ai/grok-4.6",
"name": "Grok 4.6",
"lab": "xAI",
"strict": 53.3,
"spend": 1.51,
"cost": 0.76,
"minutes": 23.9,
"tokens": 59655
},
{
"id": "google/gemini-4-argon",
"name": "Gemini 4 Argon",
"lab": "Google",
"strict": 72.3,
"spend": 2.06,
"cost": 1.04,
"minutes": 15.6,
"tokens": 43595,
"cost_estimated": true
},
{
"id": "thinkingmachines/Inkling-2-Small:peft:262144",
"name": "Inkling 2 Small",
"lab": "Thinking Machines",
"strict": 44.0,
"spend": 0.32,
"cost": 0.09,
"minutes": 15.7,
"tokens": 29743,
"cost_estimated": true
}
],
"uncapped_search_comparison": [
{
"id": "google/gemini-3.8-flash",
"name": "Gemini 3.8 Flash",
"native": 45.3,
"rows": [
{
"cond": "Exa",
"delta": -0.0,
"ci": [
-8.0,
8.0
],
"alt": 45.3
},
{
"cond": "Parallel",
"delta": 2.7,
"ci": [
-6.0,
12.0
],
"alt": 48.0
},
{
"cond": "Firecrawl",
"delta": 2.7,
"ci": [
-7.3,
12.7
],
"alt": 48.0
}
],
"abs": [
{
"cond": "Native",
"strict": 45.3,
"ci": [
34.0,
57.3
],
"mean": 84.9
},
{
"cond": "Exa",
"strict": 45.3,
"ci": [
34.0,
56.7
],
"mean": 85.8
},
{
"cond": "Parallel",
"strict": 48.0,
"ci": [
36.0,
60.0
],
"mean": 88.1
},
{
"cond": "Firecrawl",
"strict": 48.0,
"ci": [
36.0,
60.0
],
"mean": 88.2
}
]
},
{
"id": "gpt-5.6-sol",
"name": "GPT-5.6 Sol",
"native": 57.3,
"rows": [
{
"cond": "Exa",
"delta": -3.3,
"ci": [
-14.0,
6.7
],
"alt": 54.0
},
{
"cond": "Parallel",
"delta": 2.7,
"ci": [
-6.7,
12.0
],
"alt": 60.0
},
{
"cond": "Firecrawl",
"delta": -2.0,
"ci": [
-12.7,
8.7
],
"alt": 55.3
}
],
"abs": [
{
"cond": "Native",
"strict": 57.3,
"ci": [
46.0,
68.7
],
"mean": 87.0
},
{
"cond": "Exa",
"strict": 54.0,
"ci": [
42.0,
65.3
],
"mean": 87.9
},
{
"cond": "Parallel",
"strict": 60.0,
"ci": [
48.0,
71.3
],
"mean": 88.7
},
{
"cond": "Firecrawl",
"strict": 55.3,
"ci": [
43.3,
67.3
],
"mean": 87.8
}
]
},
{
"id": "moonshotai/kimi-k3",
"name": "Kimi K3",
"native": 32.0,
"rows": [
{
"cond": "Exa",
"delta": 8.0,
"ci": [
-1.3,
18.0
],
"alt": 40.0
},
{
"cond": "Parallel",
"delta": 8.7,
"ci": [
-1.3,
19.3
],
"alt": 40.7
},
{
"cond": "Firecrawl",
"delta": 0.0,
"ci": [
-9.3,
9.3
],
"alt": 32.0
}
],
"abs": [
{
"cond": "Native",
"strict": 32.0,
"ci": [
21.3,
43.3
],
"mean": 76.5
},
{
"cond": "Exa",
"strict": 40.0,
"ci": [
28.7,
51.3
],
"mean": 81.1
},
{
"cond": "Parallel",
"strict": 40.7,
"ci": [
29.3,
52.0
],
"mean": 82.2
},
{
"cond": "Firecrawl",
"strict": 32.0,
"ci": [
21.3,
43.3
],
"mean": 79.4
}
]
}
],
"failure_categories": {
"labels": [
"Scope and version",
"Retrieval and joining",
"Completeness",
"Mechanism and configuration",
"Threshold and transformation"
],
"tasks": [
39,
22,
19,
9,
11
],
"all": {
"counts": [
521,
345,
305,
169,
169
],
"total": 1509
},
"models": [
{
"id": "google/gemini-4-argon",
"name": "Gemini 4 Argon",
"counts": [
36,
16,
14,
8,
9
],
"total": 83
},
{
"id": "gpt-6-astra",
"name": "GPT-6 Astra",
"counts": [
21,
27,
18,
8,
12
],
"total": 86
},
{
"id": "anthropic/claude-opus-5.5",
"name": "Opus 5.5",
"counts": [
42,
14,
20,
5,
8
],
"total": 89
},
{
"id": "gpt-6.1-sol",
"name": "GPT-6.1 Sol",
"counts": [
21,
33,
19,
10,
13
],
"total": 96
},
{
"id": "gpt-5.6-sol",
"name": "GPT-5.6 Sol",
"counts": [
36,
30,
26,
14,
17
],
"total": 123
},
{
"id": "anthropic/claude-opus-5",
"name": "Opus 5",
"counts": [
49,
29,
26,
20,
13
],
"total": 137
},
{
"id": "x-ai/grok-4.6",
"name": "Grok 4.6",
"counts": [
42,
33,
31,
18,
16
],
"total": 140
},
{
"id": "google/gemini-3.8-flash",
"name": "Gemini 3.8 Flash",
"counts": [
57,
32,
36,
19,
21
],
"total": 165
},
{
"id": "thinkingmachines/Inkling-2-Small:peft:262144",
"name": "Inkling 2 Small",
"counts": [
54,
40,
38,
20,
16
],
"total": 168
},
{
"id": "moonshotai/kimi-k3",
"name": "Kimi K3",
"counts": [
78,
39,
37,
23,
26
],
"total": 203
},
{
"id": "anthropic/claude-sonnet-5",
"name": "Sonnet 5",
"counts": [
85,
52,
40,
24,
18
],
"total": 219
}
]
},
"judge_agreement": [
{
"key": "gpt-6-1-sol",
"name": "GPT-6.1 Sol",
"agree_whole": 94.8,
"agree_crit": 95.8,
"whole": {
"fp": 2,
"fn": 2,
"fails": 96,
"passes": 66
},
"crit": {
"fails": 199,
"fp": 15,
"fn": 9,
"passes": 886
}
},
{
"key": "gpt-5-6-sol",
"name": "GPT-5.6 Sol",
"agree_whole": 93.0,
"agree_crit": 94.9,
"whole": {
"fp": 2,
"fn": 5,
"fails": 96,
"passes": 66
},
"crit": {
"fp": 18,
"fn": 16,
"fails": 199,
"passes": 886
}
},
{
"key": "opus-5",
"name": "Opus 5",
"agree_whole": 90.1,
"agree_crit": 94.7,
"whole": {
"fp": 10,
"fn": 2,
"fails": 96,
"passes": 66
},
"crit": {
"fp": 34,
"fn": 3,
"fails": 199,
"passes": 886
}
},
{
"key": "gemini-3-7-flash",
"name": "Gemini 3.7 Flash",
"agree_whole": 89.0,
"agree_crit": 93.3,
"whole": {
"fp": 13,
"fn": 1,
"fails": 96,
"passes": 66
},
"crit": {
"fp": 46,
"fn": 6,
"fails": 199,
"passes": 886
}
},
{
"key": "kimi-k3",
"name": "Kimi K3",
"agree_whole": 84.9,
"agree_crit": 92.4,
"whole": {
"fp": 21,
"fn": 0,
"fails": 96,
"passes": 66
},
"crit": {
"fp": 54,
"fn": 8,
"fails": 199,
"passes": 886
}
}
],
"human_agreement": {
"agree_whole": 94.2,
"agree_crit": 95.8
},
"judge_study": {
"tasks": 43,
"responses": 172,
"judge_count": 5,
"human_grades_unchanged": true
},
"task_and_rubric_breakdowns": {
"task_count": 100,
"input_file_count": 435,
"file_types": [
{
"kind": "Image files",
"tasks": 25,
"files": 55
},
{
"kind": "PDF",
"tasks": 66,
"files": 138
},
{
"kind": "Audio",
"tasks": 9,
"files": 9
},
{
"kind": "Video",
"tasks": 1,
"files": 1
},
{
"kind": "Tables / spreadsheets",
"tasks": 60,
"files": 88
},
{
"kind": "Text / structured text",
"tasks": 45,
"files": 144
}
],
"image_only_pdf_files": 8,
"image_only_pdf_tasks": 4,
"image_or_image_only_pdf_tasks": 28,
"image_audio_video_union_tasks": 33,
"nontext_evidence_tasks": 30,
"nontext_evidence_rubric_items": 90,
"rubric_categories": [
{
"category": "Fact / rule identification",
"items": 217,
"item_share": 0.3273001508295626,
"passed": 6549,
"judgements": 7152,
"pass_rate": 0.9156879194630873,
"models": [
{
"model": "anthropic/claude-opus-5",
"passed": 585,
"judgements": 651,
"pass_rate": 0.8986175115207373
},
{
"model": "anthropic/claude-opus-5.5",
"passed": 629,
"judgements": 651,
"pass_rate": 0.9662058371735791
},
{
"model": "anthropic/claude-sonnet-5",
"passed": 551,
"judgements": 651,
"pass_rate": 0.8463901689708141
},
{
"model": "google/gemini-3.8-flash",
"passed": 580,
"judgements": 651,
"pass_rate": 0.890937019969278
},
{
"model": "google/gemini-4-argon",
"passed": 622,
"judgements": 651,
"pass_rate": 0.9554531490015361
},
{
"model": "gpt-5.6-sol",
"passed": 612,
"judgements": 651,
"pass_rate": 0.9400921658986175
},
{
"model": "gpt-6-astra",
"passed": 628,
"judgements": 651,
"pass_rate": 0.9646697388632872
},
{
"model": "gpt-6.1-sol",
"passed": 628,
"judgements": 651,
"pass_rate": 0.9646697388632872
},
{
"model": "moonshotai/kimi-k3",
"passed": 560,
"judgements": 651,
"pass_rate": 0.8602150537634409
},
{
"model": "x-ai/grok-4.6",
"passed": 588,
"judgements": 651,
"pass_rate": 0.9032258064516129
},
{
"model": "thinkingmachines/Inkling-2-Small:peft:262144",
"passed": 566,
"judgements": 642,
"pass_rate": 0.881619937694704
}
]
},
{
"category": "Joined application / comparison",
"items": 289,
"item_share": 0.4358974358974359,
"passed": 8198,
"judgements": 9523,
"pass_rate": 0.8608631733697364,
"models": [
{
"model": "anthropic/claude-opus-5",
"passed": 775,
"judgements": 867,
"pass_rate": 0.8938869665513264
},
{
"model": "anthropic/claude-opus-5.5",
"passed": 817,
"judgements": 867,
"pass_rate": 0.9423298731257209
},
{
"model": "anthropic/claude-sonnet-5",
"passed": 610,
"judgements": 867,
"pass_rate": 0.7035755478662054
},
{
"model": "google/gemini-3.8-flash",
"passed": 728,
"judgements": 867,
"pass_rate": 0.839677047289504
},
{
"model": "google/gemini-4-argon",
"passed": 818,
"judgements": 867,
"pass_rate": 0.9434832756632064
},
{
"model": "gpt-5.6-sol",
"passed": 757,
"judgements": 867,
"pass_rate": 0.8731257208765859
},
{
"model": "gpt-6-astra",
"passed": 812,
"judgements": 867,
"pass_rate": 0.936562860438293
},
{
"model": "gpt-6.1-sol",
"passed": 791,
"judgements": 867,
"pass_rate": 0.9123414071510957
},
{
"model": "moonshotai/kimi-k3",
"passed": 639,
"judgements": 867,
"pass_rate": 0.7370242214532872
},
{
"model": "x-ai/grok-4.6",
"passed": 750,
"judgements": 867,
"pass_rate": 0.8650519031141869
},
{
"model": "thinkingmachines/Inkling-2-Small:peft:262144",
"passed": 701,
"judgements": 853,
"pass_rate": 0.8218053927315357
}
]
},
{
"category": "Calculation / transformation",
"items": 86,
"item_share": 0.1297134238310709,
"passed": 2517,
"judgements": 2831,
"pass_rate": 0.8890851289297068,
"models": [
{
"model": "anthropic/claude-opus-5",
"passed": 236,
"judgements": 258,
"pass_rate": 0.9147286821705426
},
{
"model": "anthropic/claude-opus-5.5",
"passed": 252,
"judgements": 258,
"pass_rate": 0.9767441860465116
},
{
"model": "anthropic/claude-sonnet-5",
"passed": 190,
"judgements": 258,
"pass_rate": 0.7364341085271318
},
{
"model": "google/gemini-3.8-flash",
"passed": 233,
"judgements": 258,
"pass_rate": 0.9031007751937985
},
{
"model": "google/gemini-4-argon",
"passed": 251,
"judgements": 258,
"pass_rate": 0.9728682170542635
},
{
"model": "gpt-5.6-sol",
"passed": 220,
"judgements": 258,
"pass_rate": 0.8527131782945736
},
{
"model": "gpt-6-astra",
"passed": 236,
"judgements": 258,
"pass_rate": 0.9147286821705426
},
{
"model": "gpt-6.1-sol",
"passed": 238,
"judgements": 258,
"pass_rate": 0.9224806201550387
},
{
"model": "moonshotai/kimi-k3",
"passed": 201,
"judgements": 258,
"pass_rate": 0.7790697674418605
},
{
"model": "x-ai/grok-4.6",
"passed": 238,
"judgements": 258,
"pass_rate": 0.9224806201550387
},
{
"model": "thinkingmachines/Inkling-2-Small:peft:262144",
"passed": 222,
"judgements": 251,
"pass_rate": 0.8844621513944223
}
]
},
{
"category": "Grounding / evidence limits",
"items": 71,
"item_share": 0.10708898944193061,
"passed": 1830,
"judgements": 2338,
"pass_rate": 0.7827202737382378,
"models": [
{
"model": "anthropic/claude-opus-5",
"passed": 158,
"judgements": 213,
"pass_rate": 0.7417840375586855
},
{
"model": "anthropic/claude-opus-5.5",
"passed": 178,
"judgements": 213,
"pass_rate": 0.8356807511737089
},
{
"model": "anthropic/claude-sonnet-5",
"passed": 154,
"judgements": 213,
"pass_rate": 0.7230046948356808
},
{
"model": "google/gemini-3.8-flash",
"passed": 139,
"judgements": 213,
"pass_rate": 0.6525821596244131
},
{
"model": "google/gemini-4-argon",
"passed": 171,
"judgements": 213,
"pass_rate": 0.8028169014084507
},
{
"model": "gpt-5.6-sol",
"passed": 191,
"judgements": 213,
"pass_rate": 0.8967136150234741
},
{
"model": "gpt-6-astra",
"passed": 187,
"judgements": 213,
"pass_rate": 0.8779342723004695
},
{
"model": "gpt-6.1-sol",
"passed": 186,
"judgements": 213,
"pass_rate": 0.8732394366197183
},
{
"model": "moonshotai/kimi-k3",
"passed": 149,
"judgements": 213,
"pass_rate": 0.6995305164319249
},
{
"model": "x-ai/grok-4.6",
"passed": 165,
"judgements": 213,
"pass_rate": 0.7746478873239436
},
{
"model": "thinkingmachines/Inkling-2-Small:peft:262144",
"passed": 152,
"judgements": 208,
"pass_rate": 0.7307692307692307
}
]
}
],
"category_definitions": {
"Fact / rule identification": "Remaining criteria with only one source environment, including direct observations and rule or fact identification.",
"Joined application / comparison": "Remaining criteria supported by both workspace and web sources, including comparisons and application of public rules to the local case.",
"Calculation / transformation": "Required derived numerical values, arithmetic or numerical methods, time or unit conversions, or numerical feasibility checks.",
"Grounding / evidence limits": "Explicit source/locator requirements or requirements to limit claims to what the available evidence establishes."
},
"category_precedence": [
"Calculation / transformation",
"Grounding / evidence limits",
"Joined application / comparison",
"Fact / rule identification"
],
"provenance": "Domain counts use all author-provided submission labels; tasks may belong to multiple domains. Rubric criterionType is empty for all 663 items. Four reporting categories are exploratory post-hoc coding of primary requirement and original source-support graph, not original labels, a new grading rubric, or a causal failure taxonomy. Calculation and grounding/limits labels were reviewed against criterion text. Remainder splits on whether the linked source graph uses both workspace and web sources.",
"rate_definition": "Share of item judgments marked passed by GPT-6.1 Sol, pooled across the eleven models and three attempts per task.",
"scoring_basis": "All 3,300 native-search answers regraded with GPT-6.1 Sol against the original 663 rubric items. Rates count the new item-level judgments; task and rubric categories are unchanged.",
"inventory_definition": "Delivered environment/artifacts files excluding the standard answer_format.md. Original submission URLs without delivered local files are excluded. Image-only PDF means no extractable text and at least one embedded raster image; PDFs with an OCR layer are not identified as image-only.",
"domain_counts": {
"engineering": 54,
"public_records": 41,
"finance": 15,
"law": 14,
"other": 14,
"medicine": 13,
"science": 12,
"aviation": 4
},
"multi_domain_task_count": 55,
"domain_assignment_count": 167
},
"judging": {
"model": "gpt-6.1-sol",
"at": "2026-11-06T05:01:11.868870+00:00",
"version": "2026-11-06",
"runs": 4650,
"original_scores_preserved": true,
"human_study_context_note": "Human validation used the same system prompt and rubric but question-only task context. Full benchmark regrading retained each original full task context and reference answer.",
"valid_grades": 4644,
"inkling2_update": {
"version": "gpt61-20261108-inkling2",
"runs": 300,
"valid_grades": 294,
"null_limit_outcomes": 6
}
},
"original_generation_unchanged": true,
"original_grades_preserved": true,
"bootstrap": {
"resamples": 20000,
"seed": 20261106,
"unit": "task, preserving all three attempts"
},
"cost_note": "Original agent costs, times and tokens are unchanged; cost per correct answer uses the new grades. Three model costs use saved token counts and the original published-rate snapshot. Judging cost is separate. Inkling 2 Small costs use saved token usage at Tinker published rates on November 8, 2026, excluding judging and infrastructure.",
"inkling2_small": {
"board": {
"id": "thinkingmachines/Inkling-2-Small:peft:262144",
"name": "Inkling 2 Small",
"lab": "Thinking Machines",
"strict": 44.0,
"mean": 82.8,
"all3": 23,
"hat3": 23.0,
"counts": [
35,
21,
21,
23
]
},
"efficiency": {
"id": "thinkingmachines/Inkling-2-Small:peft:262144",
"name": "Inkling 2 Small",
"lab": "Thinking Machines",
"strict": 44.0,
"spend": 0.32,
"cost": 0.09,
"minutes": 15.7,
"tokens": 29743,
"cost_estimated": true
},
"failure_categories": {
"id": "thinkingmachines/Inkling-2-Small:peft:262144",
"name": "Inkling 2 Small",
"counts": [
54,
40,
38,
20,
16
],
"total": 168
},
"rubric_categories": {
"Joined application / comparison": {
"judgements": 853,
"passed": 701
},
"Fact / rule identification": {
"judgements": 642,
"passed": 566
},
"Grounding / evidence limits": {
"judgements": 208,
"passed": 152
},
"Calculation / transformation": {
"judgements": 251,
"passed": 222
}
},
"metrics": {
"runs": 300,
"tasks": 100,
"full_credit": 132,
"pass_at_1": 44.0,
"pass_power_3": 23.0,
"mean_rubric_score": 82.77216428571428,
"counts": [
35,
21,
21,
23
],
"pricing_source": "https://tinker-docs.thinkingmachines.ai/tinker/models.json",
"rates_per_million": {
"input": 0.3,
"cached_input": 0.03,
"output": 1.0
},
"missing_metrics": 0,
"median_minutes": 15.662570125,
"median_output_tokens": 29743.0,
"median_cost_usd": 0.08633455000000001,
"total_agent_cost_usd": 42.52916764,
"cost_per_correct": 0.32219066393939394
}
},
"excluded_model_configurations": [
"thinkingmachines/inkling"
],
"grading_versions": {
"gpt61-20261106": 4350,
"gpt61-20261108-inkling2": 300
}
}
{
"task_count": 100,
"current_native_runs": 3300,
"current_uncapped_search_runs": 1350,
"retained_run_count": 4650,
"tasks": 100,
"runs": 4650,
"native_runs": 3300,
"ablation_runs": 1350,
"current_evaluation_runs": 4650,
"native_models": [
{
"id": "google/gemini-4-argon",
"name": "Gemini 4 Argon",
"lab": "Google",
"strict": 72.3,
"strict_ci": [
64.0,
80.0
],
"mean": 94.1,
"mean_ci": [
92.0,
95.9
],
"all3": 64,
"counts": [
20,
7,
9,
64
],
"hat3": 64.0,
"hat3_ci": [
54.0,
73.0
]
},
{
"id": "gpt-6-astra",
"name": "GPT-6 Astra",
"lab": "OpenAI",
"strict": 71.3,
"strict_ci": [
63.3,
78.7
],
"mean": 92.7,
"mean_ci": [
90.3,
94.8
],
"all3": 60,
"counts": [
19,
8,
13,
60
],
"hat3": 60.0,
"hat3_ci": [
50.0,
69.0
]
},
{
"id": "anthropic/claude-opus-5.5",
"name": "Opus 5.5",
"lab": "Anthropic",
"strict": 70.3,
"strict_ci": [
62.3,
77.7
],
"mean": 94.3,
"mean_ci": [
92.3,
95.9
],
"all3": 57,
"counts": [
17,
12,
14,
57
],
"hat3": 57.0,
"hat3_ci": [
47.0,
67.0
]
},
{
"id": "gpt-6.1-sol",
"name": "GPT-6.1 Sol",
"lab": "OpenAI",
"strict": 68.0,
"strict_ci": [
59.7,
76.0