{
  "task_count": 100,
  "current_native_runs": 3300,
  "current_uncapped_search_runs": 1350,
  "retained_run_count": 4650,
  "tasks": 100,
  "runs": 4650,
  "native_runs": 3300,
  "ablation_runs": 1350,
  "current_evaluation_runs": 4650,
  "native_models": [
    {
      "id": "google/gemini-4-argon",
      "name": "Gemini 4 Argon",
      "lab": "Google",
      "strict": 72.3,
      "strict_ci": [
        64.0,
        80.0
      ],
      "mean": 94.1,
      "mean_ci": [
        92.0,
        95.9
      ],
      "all3": 64,
      "counts": [
        20,
        7,
        9,
        64
      ],
      "hat3": 64.0,
      "hat3_ci": [
        54.0,
        73.0
      ]
    },
    {
      "id": "gpt-6-astra",
      "name": "GPT-6 Astra",
      "lab": "OpenAI",
      "strict": 71.3,
      "strict_ci": [
        63.3,
        78.7
      ],
      "mean": 92.7,
      "mean_ci": [
        90.3,
        94.8
      ],
      "all3": 60,
      "counts": [
        19,
        8,
        13,
        60
      ],
      "hat3": 60.0,
      "hat3_ci": [
        50.0,
        69.0
      ]
    },
    {
      "id": "anthropic/claude-opus-5.5",
      "name": "Opus 5.5",
      "lab": "Anthropic",
      "strict": 70.3,
      "strict_ci": [
        62.3,
        77.7
      ],
      "mean": 94.3,
      "mean_ci": [
        92.3,
        95.9
      ],
      "all3": 57,
      "counts": [
        17,
        12,
        14,
        57
      ],
      "hat3": 57.0,
      "hat3_ci": [
        47.0,
        67.0
      ]
    },
    {
      "id": "gpt-6.1-sol",
      "name": "GPT-6.1 Sol",
      "lab": "OpenAI",
      "strict": 68.0,
      "strict_ci": [
        59.7,
        76.0
      ],
      "mean": 91.2,
      "mean_ci": [
        88.4,
        93.8
      ],
      "all3": 59,
      "counts": [
        21,
        13,
        7,
        59
      ],
      "hat3": 59.0,
      "hat3_ci": [
        49.0,
        68.0
      ]
    },
    {
      "id": "gpt-5.6-sol",
      "name": "GPT-5.6 Sol",
      "lab": "OpenAI",
      "strict": 59.0,
      "strict_ci": [
        51.0,
        66.7
      ],
      "mean": 88.1,
      "mean_ci": [
        84.6,
        91.1
      ],
      "all3": 38,
      "counts": [
        22,
        17,
        23,
        38
      ],
      "hat3": 38.0,
      "hat3_ci": [
        29.0,
        48.0
      ]
    },
    {
      "id": "anthropic/claude-opus-5",
      "name": "Opus 5",
      "lab": "Anthropic",
      "strict": 54.3,
      "strict_ci": [
        46.3,
        62.3
      ],
      "mean": 88.2,
      "mean_ci": [
        85.3,
        90.9
      ],
      "all3": 34,
      "counts": [
        29,
        13,
        24,
        34
      ],
      "hat3": 34.0,
      "hat3_ci": [
        25.0,
        44.0
      ]
    },
    {
      "id": "x-ai/grok-4.6",
      "name": "Grok 4.6",
      "lab": "xAI",
      "strict": 53.3,
      "strict_ci": [
        45.0,
        61.3
      ],
      "mean": 87.3,
      "mean_ci": [
        84.2,
        90.2
      ],
      "all3": 35,
      "counts": [
        31,
        13,
        21,
        35
      ],
      "hat3": 35.0,
      "hat3_ci": [
        26.0,
        44.0
      ]
    },
    {
      "id": "google/gemini-3.8-flash",
      "name": "Gemini 3.8 Flash",
      "lab": "Google",
      "strict": 45.0,
      "strict_ci": [
        37.0,
        53.3
      ],
      "mean": 83.0,
      "mean_ci": [
        79.1,
        86.6
      ],
      "all3": 29,
      "counts": [
        37,
        20,
        14,
        29
      ],
      "hat3": 29.0,
      "hat3_ci": [
        20.0,
        38.0
      ]
    },
    {
      "id": "thinkingmachines/Inkling-2-Small:peft:262144",
      "name": "Inkling 2 Small",
      "lab": "Thinking Machines",
      "strict": 44.0,
      "mean": 82.8,
      "all3": 23,
      "hat3": 23.0,
      "counts": [
        35,
        21,
        21,
        23
      ]
    },
    {
      "id": "moonshotai/kimi-k3",
      "name": "Kimi K3",
      "lab": "Moonshot AI",
      "strict": 32.3,
      "strict_ci": [
        25.0,
        39.7
      ],
      "mean": 76.6,
      "mean_ci": [
        72.4,
        80.5
      ],
      "all3": 15,
      "counts": [
        49,
        20,
        16,
        15
      ],
      "hat3": 15.0,
      "hat3_ci": [
        8.0,
        22.0
      ]
    },
    {
      "id": "anthropic/claude-sonnet-5",
      "name": "Sonnet 5",
      "lab": "Anthropic",
      "strict": 27.0,
      "strict_ci": [
        21.0,
        33.3
      ],
      "mean": 72.0,
      "mean_ci": [
        67.5,
        76.3
      ],
      "all3": 7,
      "counts": [
        51,
        24,
        18,
        7
      ],
      "hat3": 7.0,
      "hat3_ci": [
        2.0,
        12.0
      ]
    }
  ],
  "efficiency": [
    {
      "id": "anthropic/claude-opus-5",
      "name": "Opus 5",
      "lab": "Anthropic",
      "strict": 54.3,
      "spend": 3.3,
      "cost": 1.43,
      "minutes": 30.1,
      "tokens": 74856
    },
    {
      "id": "anthropic/claude-opus-5.5",
      "name": "Opus 5.5",
      "lab": "Anthropic",
      "strict": 70.3,
      "spend": 1.31,
      "cost": 0.85,
      "minutes": 15.5,
      "tokens": 42376
    },
    {
      "id": "anthropic/claude-sonnet-5",
      "name": "Sonnet 5",
      "lab": "Anthropic",
      "strict": 27.0,
      "spend": 3.04,
      "cost": 0.54,
      "minutes": 17.7,
      "tokens": 60656
    },
    {
      "id": "google/gemini-3.8-flash",
      "name": "Gemini 3.8 Flash",
      "lab": "Google",
      "strict": 45.0,
      "spend": 2.49,
      "cost": 0.85,
      "minutes": 14.2,
      "tokens": 55176,
      "cost_estimated": true
    },
    {
      "id": "gpt-5.6-sol",
      "name": "GPT-5.6 Sol",
      "lab": "OpenAI",
      "strict": 59.0,
      "spend": 2.68,
      "cost": 1.21,
      "minutes": 12.2,
      "tokens": 30446
    },
    {
      "id": "gpt-6-astra",
      "name": "GPT-6 Astra",
      "lab": "OpenAI",
      "strict": 71.3,
      "spend": 5.22,
      "cost": 2.51,
      "minutes": 10.0,
      "tokens": 21260
    },
    {
      "id": "gpt-6.1-sol",
      "name": "GPT-6.1 Sol",
      "lab": "OpenAI",
      "strict": 68.0,
      "spend": 0.98,
      "cost": 0.48,
      "minutes": 7.6,
      "tokens": 22432
    },
    {
      "id": "moonshotai/kimi-k3",
      "name": "Kimi K3",
      "lab": "Moonshot AI",
      "strict": 32.3,
      "spend": 0.85,
      "cost": 0.2,
      "minutes": 14.5,
      "tokens": 14840,
      "cost_estimated": true
    },
    {
      "id": "x-ai/grok-4.6",
      "name": "Grok 4.6",
      "lab": "xAI",
      "strict": 53.3,
      "spend": 1.51,
      "cost": 0.76,
      "minutes": 23.9,
      "tokens": 59655
    },
    {
      "id": "google/gemini-4-argon",
      "name": "Gemini 4 Argon",
      "lab": "Google",
      "strict": 72.3,
      "spend": 2.06,
      "cost": 1.04,
      "minutes": 15.6,
      "tokens": 43595,
      "cost_estimated": true
    },
    {
      "id": "thinkingmachines/Inkling-2-Small:peft:262144",
      "name": "Inkling 2 Small",
      "lab": "Thinking Machines",
      "strict": 44.0,
      "spend": 0.32,
      "cost": 0.09,
      "minutes": 15.7,
      "tokens": 29743,
      "cost_estimated": true
    }
  ],
  "uncapped_search_comparison": [
    {
      "id": "google/gemini-3.8-flash",
      "name": "Gemini 3.8 Flash",
      "native": 45.3,
      "rows": [
        {
          "cond": "Exa",
          "delta": -0.0,
          "ci": [
            -8.0,
            8.0
          ],
          "alt": 45.3
        },
        {
          "cond": "Parallel",
          "delta": 2.7,
          "ci": [
            -6.0,
            12.0
          ],
          "alt": 48.0
        },
        {
          "cond": "Firecrawl",
          "delta": 2.7,
          "ci": [
            -7.3,
            12.7
          ],
          "alt": 48.0
        }
      ],
      "abs": [
        {
          "cond": "Native",
          "strict": 45.3,
          "ci": [
            34.0,
            57.3
          ],
          "mean": 84.9
        },
        {
          "cond": "Exa",
          "strict": 45.3,
          "ci": [
            34.0,
            56.7
          ],
          "mean": 85.8
        },
        {
          "cond": "Parallel",
          "strict": 48.0,
          "ci": [
            36.0,
            60.0
          ],
          "mean": 88.1
        },
        {
          "cond": "Firecrawl",
          "strict": 48.0,
          "ci": [
            36.0,
            60.0
          ],
          "mean": 88.2
        }
      ]
    },
    {
      "id": "gpt-5.6-sol",
      "name": "GPT-5.6 Sol",
      "native": 57.3,
      "rows": [
        {
          "cond": "Exa",
          "delta": -3.3,
          "ci": [
            -14.0,
            6.7
          ],
          "alt": 54.0
        },
        {
          "cond": "Parallel",
          "delta": 2.7,
          "ci": [
            -6.7,
            12.0
          ],
          "alt": 60.0
        },
        {
          "cond": "Firecrawl",
          "delta": -2.0,
          "ci": [
            -12.7,
            8.7
          ],
          "alt": 55.3
        }
      ],
      "abs": [
        {
          "cond": "Native",
          "strict": 57.3,
          "ci": [
            46.0,
            68.7
          ],
          "mean": 87.0
        },
        {
          "cond": "Exa",
          "strict": 54.0,
          "ci": [
            42.0,
            65.3
          ],
          "mean": 87.9
        },
        {
          "cond": "Parallel",
          "strict": 60.0,
          "ci": [
            48.0,
            71.3
          ],
          "mean": 88.7
        },
        {
          "cond": "Firecrawl",
          "strict": 55.3,
          "ci": [
            43.3,
            67.3
          ],
          "mean": 87.8
        }
      ]
    },
    {
      "id": "moonshotai/kimi-k3",
      "name": "Kimi K3",
      "native": 32.0,
      "rows": [
        {
          "cond": "Exa",
          "delta": 8.0,
          "ci": [
            -1.3,
            18.0
          ],
          "alt": 40.0
        },
        {
          "cond": "Parallel",
          "delta": 8.7,
          "ci": [
            -1.3,
            19.3
          ],
          "alt": 40.7
        },
        {
          "cond": "Firecrawl",
          "delta": 0.0,
          "ci": [
            -9.3,
            9.3
          ],
          "alt": 32.0
        }
      ],
      "abs": [
        {
          "cond": "Native",
          "strict": 32.0,
          "ci": [
            21.3,
            43.3
          ],
          "mean": 76.5
        },
        {
          "cond": "Exa",
          "strict": 40.0,
          "ci": [
            28.7,
            51.3
          ],
          "mean": 81.1
        },
        {
          "cond": "Parallel",
          "strict": 40.7,
          "ci": [
            29.3,
            52.0
          ],
          "mean": 82.2
        },
        {
          "cond": "Firecrawl",
          "strict": 32.0,
          "ci": [
            21.3,
            43.3
          ],
          "mean": 79.4
        }
      ]
    }
  ],
  "failure_categories": {
    "labels": [
      "Scope and version",
      "Retrieval and joining",
      "Completeness",
      "Mechanism and configuration",
      "Threshold and transformation"
    ],
    "tasks": [
      39,
      22,
      19,
      9,
      11
    ],
    "all": {
      "counts": [
        521,
        345,
        305,
        169,
        169
      ],
      "total": 1509
    },
    "models": [
      {
        "id": "google/gemini-4-argon",
        "name": "Gemini 4 Argon",
        "counts": [
          36,
          16,
          14,
          8,
          9
        ],
        "total": 83
      },
      {
        "id": "gpt-6-astra",
        "name": "GPT-6 Astra",
        "counts": [
          21,
          27,
          18,
          8,
          12
        ],
        "total": 86
      },
      {
        "id": "anthropic/claude-opus-5.5",
        "name": "Opus 5.5",
        "counts": [
          42,
          14,
          20,
          5,
          8
        ],
        "total": 89
      },
      {
        "id": "gpt-6.1-sol",
        "name": "GPT-6.1 Sol",
        "counts": [
          21,
          33,
          19,
          10,
          13
        ],
        "total": 96
      },
      {
        "id": "gpt-5.6-sol",
        "name": "GPT-5.6 Sol",
        "counts": [
          36,
          30,
          26,
          14,
          17
        ],
        "total": 123
      },
      {
        "id": "anthropic/claude-opus-5",
        "name": "Opus 5",
        "counts": [
          49,
          29,
          26,
          20,
          13
        ],
        "total": 137
      },
      {
        "id": "x-ai/grok-4.6",
        "name": "Grok 4.6",
        "counts": [
          42,
          33,
          31,
          18,
          16
        ],
        "total": 140
      },
      {
        "id": "google/gemini-3.8-flash",
        "name": "Gemini 3.8 Flash",
        "counts": [
          57,
          32,
          36,
          19,
          21
        ],
        "total": 165
      },
      {
        "id": "thinkingmachines/Inkling-2-Small:peft:262144",
        "name": "Inkling 2 Small",
        "counts": [
          54,
          40,
          38,
          20,
          16
        ],
        "total": 168
      },
      {
        "id": "moonshotai/kimi-k3",
        "name": "Kimi K3",
        "counts": [
          78,
          39,
          37,
          23,
          26
        ],
        "total": 203
      },
      {
        "id": "anthropic/claude-sonnet-5",
        "name": "Sonnet 5",
        "counts": [
          85,
          52,
          40,
          24,
          18
        ],
        "total": 219
      }
    ]
  },
  "judge_agreement": [
    {
      "key": "gpt-6-1-sol",
      "name": "GPT-6.1 Sol",
      "agree_whole": 94.8,
      "agree_crit": 95.8,
      "whole": {
        "fp": 2,
        "fn": 2,
        "fails": 96,
        "passes": 66
      },
      "crit": {
        "fails": 199,
        "fp": 15,
        "fn": 9,
        "passes": 886
      }
    },
    {
      "key": "gpt-5-6-sol",
      "name": "GPT-5.6 Sol",
      "agree_whole": 93.0,
      "agree_crit": 94.9,
      "whole": {
        "fp": 2,
        "fn": 5,
        "fails": 96,
        "passes": 66
      },
      "crit": {
        "fp": 18,
        "fn": 16,
        "fails": 199,
        "passes": 886
      }
    },
    {
      "key": "opus-5",
      "name": "Opus 5",
      "agree_whole": 90.1,
      "agree_crit": 94.7,
      "whole": {
        "fp": 10,
        "fn": 2,
        "fails": 96,
        "passes": 66
      },
      "crit": {
        "fp": 34,
        "fn": 3,
        "fails": 199,
        "passes": 886
      }
    },
    {
      "key": "gemini-3-7-flash",
      "name": "Gemini 3.7 Flash",
      "agree_whole": 89.0,
      "agree_crit": 93.3,
      "whole": {
        "fp": 13,
        "fn": 1,
        "fails": 96,
        "passes": 66
      },
      "crit": {
        "fp": 46,
        "fn": 6,
        "fails": 199,
        "passes": 886
      }
    },
    {
      "key": "kimi-k3",
      "name": "Kimi K3",
      "agree_whole": 84.9,
      "agree_crit": 92.4,
      "whole": {
        "fp": 21,
        "fn": 0,
        "fails": 96,
        "passes": 66
      },
      "crit": {
        "fp": 54,
        "fn": 8,
        "fails": 199,
        "passes": 886
      }
    }
  ],
  "human_agreement": {
    "agree_whole": 94.2,
    "agree_crit": 95.8
  },
  "judge_study": {
    "tasks": 43,
    "responses": 172,
    "judge_count": 5,
    "human_grades_unchanged": true
  },
  "task_and_rubric_breakdowns": {
    "task_count": 100,
    "input_file_count": 435,
    "file_types": [
      {
        "kind": "Image files",
        "tasks": 25,
        "files": 55
      },
      {
        "kind": "PDF",
        "tasks": 66,
        "files": 138
      },
      {
        "kind": "Audio",
        "tasks": 9,
        "files": 9
      },
      {
        "kind": "Video",
        "tasks": 1,
        "files": 1
      },
      {
        "kind": "Tables / spreadsheets",
        "tasks": 60,
        "files": 88
      },
      {
        "kind": "Text / structured text",
        "tasks": 45,
        "files": 144
      }
    ],
    "image_only_pdf_files": 8,
    "image_only_pdf_tasks": 4,
    "image_or_image_only_pdf_tasks": 28,
    "image_audio_video_union_tasks": 33,
    "nontext_evidence_tasks": 30,
    "nontext_evidence_rubric_items": 90,
    "rubric_categories": [
      {
        "category": "Fact / rule identification",
        "items": 217,
        "item_share": 0.3273001508295626,
        "passed": 6549,
        "judgements": 7152,
        "pass_rate": 0.9156879194630873,
        "models": [
          {
            "model": "anthropic/claude-opus-5",
            "passed": 585,
            "judgements": 651,
            "pass_rate": 0.8986175115207373
          },
          {
            "model": "anthropic/claude-opus-5.5",
            "passed": 629,
            "judgements": 651,
            "pass_rate": 0.9662058371735791
          },
          {
            "model": "anthropic/claude-sonnet-5",
            "passed": 551,
            "judgements": 651,
            "pass_rate": 0.8463901689708141
          },
          {
            "model": "google/gemini-3.8-flash",
            "passed": 580,
            "judgements": 651,
            "pass_rate": 0.890937019969278
          },
          {
            "model": "google/gemini-4-argon",
            "passed": 622,
            "judgements": 651,
            "pass_rate": 0.9554531490015361
          },
          {
            "model": "gpt-5.6-sol",
            "passed": 612,
            "judgements": 651,
            "pass_rate": 0.9400921658986175
          },
          {
            "model": "gpt-6-astra",
            "passed": 628,
            "judgements": 651,
            "pass_rate": 0.9646697388632872
          },
          {
            "model": "gpt-6.1-sol",
            "passed": 628,
            "judgements": 651,
            "pass_rate": 0.9646697388632872
          },
          {
            "model": "moonshotai/kimi-k3",
            "passed": 560,
            "judgements": 651,
            "pass_rate": 0.8602150537634409
          },
          {
            "model": "x-ai/grok-4.6",
            "passed": 588,
            "judgements": 651,
            "pass_rate": 0.9032258064516129
          },
          {
            "model": "thinkingmachines/Inkling-2-Small:peft:262144",
            "passed": 566,
            "judgements": 642,
            "pass_rate": 0.881619937694704
          }
        ]
      },
      {
        "category": "Joined application / comparison",
        "items": 289,
        "item_share": 0.4358974358974359,
        "passed": 8198,
        "judgements": 9523,
        "pass_rate": 0.8608631733697364,
        "models": [
          {
            "model": "anthropic/claude-opus-5",
            "passed": 775,
            "judgements": 867,
            "pass_rate": 0.8938869665513264
          },
          {
            "model": "anthropic/claude-opus-5.5",
            "passed": 817,
            "judgements": 867,
            "pass_rate": 0.9423298731257209
          },
          {
            "model": "anthropic/claude-sonnet-5",
            "passed": 610,
            "judgements": 867,
            "pass_rate": 0.7035755478662054
          },
          {
            "model": "google/gemini-3.8-flash",
            "passed": 728,
            "judgements": 867,
            "pass_rate": 0.839677047289504
          },
          {
            "model": "google/gemini-4-argon",
            "passed": 818,
            "judgements": 867,
            "pass_rate": 0.9434832756632064
          },
          {
            "model": "gpt-5.6-sol",
            "passed": 757,
            "judgements": 867,
            "pass_rate": 0.8731257208765859
          },
          {
            "model": "gpt-6-astra",
            "passed": 812,
            "judgements": 867,
            "pass_rate": 0.936562860438293
          },
          {
            "model": "gpt-6.1-sol",
            "passed": 791,
            "judgements": 867,
            "pass_rate": 0.9123414071510957
          },
          {
            "model": "moonshotai/kimi-k3",
            "passed": 639,
            "judgements": 867,
            "pass_rate": 0.7370242214532872
          },
          {
            "model": "x-ai/grok-4.6",
            "passed": 750,
            "judgements": 867,
            "pass_rate": 0.8650519031141869
          },
          {
            "model": "thinkingmachines/Inkling-2-Small:peft:262144",
            "passed": 701,
            "judgements": 853,
            "pass_rate": 0.8218053927315357
          }
        ]
      },
      {
        "category": "Calculation / transformation",
        "items": 86,
        "item_share": 0.1297134238310709,
        "passed": 2517,
        "judgements": 2831,
        "pass_rate": 0.8890851289297068,
        "models": [
          {
            "model": "anthropic/claude-opus-5",
            "passed": 236,
            "judgements": 258,
            "pass_rate": 0.9147286821705426
          },
          {
            "model": "anthropic/claude-opus-5.5",
            "passed": 252,
            "judgements": 258,
            "pass_rate": 0.9767441860465116
          },
          {
            "model": "anthropic/claude-sonnet-5",
            "passed": 190,
            "judgements": 258,
            "pass_rate": 0.7364341085271318
          },
          {
            "model": "google/gemini-3.8-flash",
            "passed": 233,
            "judgements": 258,
            "pass_rate": 0.9031007751937985
          },
          {
            "model": "google/gemini-4-argon",
            "passed": 251,
            "judgements": 258,
            "pass_rate": 0.9728682170542635
          },
          {
            "model": "gpt-5.6-sol",
            "passed": 220,
            "judgements": 258,
            "pass_rate": 0.8527131782945736
          },
          {
            "model": "gpt-6-astra",
            "passed": 236,
            "judgements": 258,
            "pass_rate": 0.9147286821705426
          },
          {
            "model": "gpt-6.1-sol",
            "passed": 238,
            "judgements": 258,
            "pass_rate": 0.9224806201550387
          },
          {
            "model": "moonshotai/kimi-k3",
            "passed": 201,
            "judgements": 258,
            "pass_rate": 0.7790697674418605
          },
          {
            "model": "x-ai/grok-4.6",
            "passed": 238,
            "judgements": 258,
            "pass_rate": 0.9224806201550387
          },
          {
            "model": "thinkingmachines/Inkling-2-Small:peft:262144",
            "passed": 222,
            "judgements": 251,
            "pass_rate": 0.8844621513944223
          }
        ]
      },
      {
        "category": "Grounding / evidence limits",
        "items": 71,
        "item_share": 0.10708898944193061,
        "passed": 1830,
        "judgements": 2338,
        "pass_rate": 0.7827202737382378,
        "models": [
          {
            "model": "anthropic/claude-opus-5",
            "passed": 158,
            "judgements": 213,
            "pass_rate": 0.7417840375586855
          },
          {
            "model": "anthropic/claude-opus-5.5",
            "passed": 178,
            "judgements": 213,
            "pass_rate": 0.8356807511737089
          },
          {
            "model": "anthropic/claude-sonnet-5",
            "passed": 154,
            "judgements": 213,
            "pass_rate": 0.7230046948356808
          },
          {
            "model": "google/gemini-3.8-flash",
            "passed": 139,
            "judgements": 213,
            "pass_rate": 0.6525821596244131
          },
          {
            "model": "google/gemini-4-argon",
            "passed": 171,
            "judgements": 213,
            "pass_rate": 0.8028169014084507
          },
          {
            "model": "gpt-5.6-sol",
            "passed": 191,
            "judgements": 213,
            "pass_rate": 0.8967136150234741
          },
          {
            "model": "gpt-6-astra",
            "passed": 187,
            "judgements": 213,
            "pass_rate": 0.8779342723004695
          },
          {
            "model": "gpt-6.1-sol",
            "passed": 186,
            "judgements": 213,
            "pass_rate": 0.8732394366197183
          },
          {
            "model": "moonshotai/kimi-k3",
            "passed": 149,
            "judgements": 213,
            "pass_rate": 0.6995305164319249
          },
          {
            "model": "x-ai/grok-4.6",
            "passed": 165,
            "judgements": 213,
            "pass_rate": 0.7746478873239436
          },
          {
            "model": "thinkingmachines/Inkling-2-Small:peft:262144",
            "passed": 152,
            "judgements": 208,
            "pass_rate": 0.7307692307692307
          }
        ]
      }
    ],
    "category_definitions": {
      "Fact / rule identification": "Remaining criteria with only one source environment, including direct observations and rule or fact identification.",
      "Joined application / comparison": "Remaining criteria supported by both workspace and web sources, including comparisons and application of public rules to the local case.",
      "Calculation / transformation": "Required derived numerical values, arithmetic or numerical methods, time or unit conversions, or numerical feasibility checks.",
      "Grounding / evidence limits": "Explicit source/locator requirements or requirements to limit claims to what the available evidence establishes."
    },
    "category_precedence": [
      "Calculation / transformation",
      "Grounding / evidence limits",
      "Joined application / comparison",
      "Fact / rule identification"
    ],
    "provenance": "Domain counts use all author-provided submission labels; tasks may belong to multiple domains. Rubric criterionType is empty for all 663 items. Four reporting categories are exploratory post-hoc coding of primary requirement and original source-support graph, not original labels, a new grading rubric, or a causal failure taxonomy. Calculation and grounding/limits labels were reviewed against criterion text. Remainder splits on whether the linked source graph uses both workspace and web sources.",
    "rate_definition": "Share of item judgments marked passed by GPT-6.1 Sol, pooled across the eleven models and three attempts per task.",
    "scoring_basis": "All 3,300 native-search answers regraded with GPT-6.1 Sol against the original 663 rubric items. Rates count the new item-level judgments; task and rubric categories are unchanged.",
    "inventory_definition": "Delivered environment/artifacts files excluding the standard answer_format.md. Original submission URLs without delivered local files are excluded. Image-only PDF means no extractable text and at least one embedded raster image; PDFs with an OCR layer are not identified as image-only.",
    "domain_counts": {
      "engineering": 54,
      "public_records": 41,
      "finance": 15,
      "law": 14,
      "other": 14,
      "medicine": 13,
      "science": 12,
      "aviation": 4
    },
    "multi_domain_task_count": 55,
    "domain_assignment_count": 167
  },
  "judging": {
    "model": "gpt-6.1-sol",
    "at": "2026-11-06T05:01:11.868870+00:00",
    "version": "2026-11-06",
    "runs": 4650,
    "original_scores_preserved": true,
    "human_study_context_note": "Human validation used the same system prompt and rubric but question-only task context. Full benchmark regrading retained each original full task context and reference answer.",
    "valid_grades": 4644,
    "inkling2_update": {
      "version": "gpt61-20261108-inkling2",
      "runs": 300,
      "valid_grades": 294,
      "null_limit_outcomes": 6
    }
  },
  "original_generation_unchanged": true,
  "original_grades_preserved": true,
  "bootstrap": {
    "resamples": 20000,
    "seed": 20261106,
    "unit": "task, preserving all three attempts"
  },
  "cost_note": "Original agent costs, times and tokens are unchanged; cost per correct answer uses the new grades. Three model costs use saved token counts and the original published-rate snapshot. Judging cost is separate. Inkling 2 Small costs use saved token usage at Tinker published rates on November 8, 2026, excluding judging and infrastructure.",
  "inkling2_small": {
    "board": {
      "id": "thinkingmachines/Inkling-2-Small:peft:262144",
      "name": "Inkling 2 Small",
      "lab": "Thinking Machines",
      "strict": 44.0,
      "mean": 82.8,
      "all3": 23,
      "hat3": 23.0,
      "counts": [
        35,
        21,
        21,
        23
      ]
    },
    "efficiency": {
      "id": "thinkingmachines/Inkling-2-Small:peft:262144",
      "name": "Inkling 2 Small",
      "lab": "Thinking Machines",
      "strict": 44.0,
      "spend": 0.32,
      "cost": 0.09,
      "minutes": 15.7,
      "tokens": 29743,
      "cost_estimated": true
    },
    "failure_categories": {
      "id": "thinkingmachines/Inkling-2-Small:peft:262144",
      "name": "Inkling 2 Small",
      "counts": [
        54,
        40,
        38,
        20,
        16
      ],
      "total": 168
    },
    "rubric_categories": {
      "Joined application / comparison": {
        "judgements": 853,
        "passed": 701
      },
      "Fact / rule identification": {
        "judgements": 642,
        "passed": 566
      },
      "Grounding / evidence limits": {
        "judgements": 208,
        "passed": 152
      },
      "Calculation / transformation": {
        "judgements": 251,
        "passed": 222
      }
    },
    "metrics": {
      "runs": 300,
      "tasks": 100,
      "full_credit": 132,
      "pass_at_1": 44.0,
      "pass_power_3": 23.0,
      "mean_rubric_score": 82.77216428571428,
      "counts": [
        35,
        21,
        21,
        23
      ],
      "pricing_source": "https://tinker-docs.thinkingmachines.ai/tinker/models.json",
      "rates_per_million": {
        "input": 0.3,
        "cached_input": 0.03,
        "output": 1.0
      },
      "missing_metrics": 0,
      "median_minutes": 15.662570125,
      "median_output_tokens": 29743.0,
      "median_cost_usd": 0.08633455000000001,
      "total_agent_cost_usd": 42.52916764,
      "cost_per_correct": 0.32219066393939394
    }
  },
  "excluded_model_configurations": [
    "thinkingmachines/inkling"
  ],
  "grading_versions": {
    "gpt61-20261106": 4350,
    "gpt61-20261108-inkling2": 300
  }
}