| {"scope":{"tasks":100,"models":10,"native_runs":3000,"attempts_per_task_model":3,"criterion_count":663,"criterion_decisions":19890,"categories":{"both":390,"web":121,"local":152}},"method":{"classification":"Union of frozen supportingClaimIds categories: workspace_fact=local; web_fact=web; union=both. No inferred labels or model-generated labels.","accuracy":"Passed criterion decisions divided by all criterion decisions within each source group. Every required criterion has unit weight.","perfect_runs":"Original validated reward1.0 implies all positive-weight criteria passed; frozen contract and validation consumed.","confidence_interval":"Percentile 95% task-cluster bootstrap; 10000 resamples of 100 tasks with replacement, same draw for each model and category; three final attempts kept together.","seed":20261013,"new_model_calls":0,"new_grading_calls":0,"raw_archive_reads":0},"models":[{"model":"anthropic/claude-opus-5.5","name":"Opus 5.5","categories":{"local":{"passed":442,"total":456,"accuracy":0.9692982456140351,"ci95":[0.9406938958455893,0.9901960784313726],"task_count":63,"task_mean_accuracy":0.9594356261022927},"web":{"passed":355,"total":363,"accuracy":0.977961432506887,"ci95":[0.951280487804878,0.997289972899729],"task_count":59,"task_mean_accuracy":0.9755178907721281},"both":{"passed":1140,"total":1170,"accuracy":0.9743589743589743,"ci95":[0.9581195462478186,0.9873741851135468],"task_count":100,"task_mean_accuracy":0.9720185185185186}}},{"model":"gpt-6-astra","name":"GPT-6 Astra","categories":{"local":{"passed":442,"total":456,"accuracy":0.9692982456140351,"ci95":[0.9401197604790419,0.993103448275862],"task_count":63,"task_mean_accuracy":0.9653880070546736},"web":{"passed":355,"total":363,"accuracy":0.977961432506887,"ci95":[0.9534307136833229,0.9971014492753624],"task_count":59,"task_mean_accuracy":0.9759887005649718},"both":{"passed":1123,"total":1170,"accuracy":0.9598290598290599,"ci95":[0.935357383642535,0.9808027923211169],"task_count":100,"task_mean_accuracy":0.9469166666666666}}},{"model":"gpt-6.1-sol","name":"GPT-6.1 Sol","categories":{"local":{"passed":437,"total":456,"accuracy":0.9583333333333334,"ci95":[0.9288582732525438,0.9832134292565947],"task_count":63,"task_mean_accuracy":0.9543650793650794},"web":{"passed":354,"total":363,"accuracy":0.9752066115702479,"ci95":[0.946031746031746,0.9971264367816092],"task_count":59,"task_mean_accuracy":0.9675141242937854},"both":{"passed":1105,"total":1170,"accuracy":0.9444444444444444,"ci95":[0.9154923961632635,0.9698536890428281],"task_count":100,"task_mean_accuracy":0.9302777777777778}}},{"model":"anthropic/claude-opus-5","name":"Opus 5","categories":{"local":{"passed":434,"total":456,"accuracy":0.9517543859649122,"ci95":[0.9171717171717172,0.9791666666666666],"task_count":63,"task_mean_accuracy":0.9545855379188712},"web":{"passed":329,"total":363,"accuracy":0.90633608815427,"ci95":[0.8526309822174484,0.9538475177304965],"task_count":59,"task_mean_accuracy":0.8973634651600753},"both":{"passed":1074,"total":1170,"accuracy":0.9179487179487179,"ci95":[0.8908923346923231,0.941871869728818],"task_count":100,"task_mean_accuracy":0.9108108465608464}}},{"model":"x-ai/grok-4.6","name":"Grok 4.6","categories":{"local":{"passed":432,"total":456,"accuracy":0.9473684210526315,"ci95":[0.9125665983606558,0.9750011140819964],"task_count":63,"task_mean_accuracy":0.9320987654320988},"web":{"passed":325,"total":363,"accuracy":0.8953168044077136,"ci95":[0.847348779854621,0.9391370223978919],"task_count":59,"task_mean_accuracy":0.8894538606403013},"both":{"passed":1070,"total":1170,"accuracy":0.9145299145299145,"ci95":[0.8847176533101466,0.9410257985257985],"task_count":100,"task_mean_accuracy":0.9074457671957671}}},{"model":"gpt-5.6-sol","name":"GPT-5.6 Sol","categories":{"local":{"passed":433,"total":456,"accuracy":0.9495614035087719,"ci95":[0.9185171862509992,0.9761904761904762],"task_count":63,"task_mean_accuracy":0.9506172839506174},"web":{"passed":349,"total":363,"accuracy":0.9614325068870524,"ci95":[0.9338619289955168,0.9838709677419355],"task_count":59,"task_mean_accuracy":0.9576271186440678},"both":{"passed":1050,"total":1170,"accuracy":0.8974358974358975,"ci95":[0.8622446521961386,0.9289347036143242],"task_count":100,"task_mean_accuracy":0.8769880952380952}}},{"model":"google/gemini-3.8-flash","name":"Gemini 3.8 Flash","categories":{"local":{"passed":424,"total":456,"accuracy":0.9298245614035088,"ci95":[0.8941441441441441,0.9612403100775194],"task_count":63,"task_mean_accuracy":0.9243197278911566},"web":{"passed":311,"total":363,"accuracy":0.8567493112947658,"ci95":[0.8042318792318792,0.9066666666666666],"task_count":59,"task_mean_accuracy":0.8516949152542372},"both":{"passed":1031,"total":1170,"accuracy":0.8811965811965812,"ci95":[0.8472213476070529,0.9122807017543859],"task_count":100,"task_mean_accuracy":0.8623981481481482}}},{"model":"moonshotai/kimi-k3","name":"Kimi K3","categories":{"local":{"passed":414,"total":456,"accuracy":0.9078947368421053,"ci95":[0.8719361147327249,0.9396135265700483],"task_count":63,"task_mean_accuracy":0.9066830435878054},"web":{"passed":303,"total":363,"accuracy":0.8347107438016529,"ci95":[0.764102564102564,0.8948717948717949],"task_count":59,"task_mean_accuracy":0.8244821092278719},"both":{"passed":919,"total":1170,"accuracy":0.7854700854700855,"ci95":[0.737835695926474,0.8280106405106405],"task_count":100,"task_mean_accuracy":0.7562984607984609}}},{"model":"anthropic/claude-sonnet-5","name":"Sonnet 5","categories":{"local":{"passed":409,"total":456,"accuracy":0.8969298245614035,"ci95":[0.8524579735535811,0.9346846846846847],"task_count":63,"task_mean_accuracy":0.8816137566137567},"web":{"passed":293,"total":363,"accuracy":0.8071625344352618,"ci95":[0.7418540223106787,0.8661629045557616],"task_count":59,"task_mean_accuracy":0.7684557438794727},"both":{"passed":857,"total":1170,"accuracy":0.7324786324786324,"ci95":[0.6759516589683016,0.7837439052371938],"task_count":100,"task_mean_accuracy":0.6883015873015873}}},{"model":"thinkingmachines/inkling","name":"Inkling","categories":{"local":{"passed":322,"total":456,"accuracy":0.706140350877193,"ci95":[0.6293327586206896,0.7783791461916463],"task_count":63,"task_mean_accuracy":0.7074829931972789},"web":{"passed":142,"total":363,"accuracy":0.39118457300275483,"ci95":[0.31578615098351936,0.46264367816091956],"task_count":59,"task_mean_accuracy":0.34651600753295664},"both":{"passed":441,"total":1170,"accuracy":0.3769230769230769,"ci95":[0.3139412208449366,0.43792640790493415],"task_count":100,"task_mean_accuracy":0.3424011544011544}}}]} |