{"families":["tinanta","krdanta","subanta","taddhitanta"],"figures":["efficiency"],"models":[{"id":"astra-xhigh","label":"GPT-6 Astra","short":"Astra","provider":"openai","scopes":{"tinanta":{"n":5000,"correct":4679,"accuracy":93.58,"tokens":{"n":5000,"median":876.5},"cost":{"n":5000,"median":0.046105}},"krdanta":{"n":5000,"correct":4806,"accuracy":96.12,"tokens":{"n":5000,"median":589.5},"cost":{"n":5000,"median":0.03183}},"subanta":{"n":5000,"correct":4926,"accuracy":98.52,"tokens":{"n":5000,"median":78.0},"cost":{"n":5000,"median":0.00555}},"taddhitanta":{"n":5000,"correct":4280,"accuracy":85.6,"tokens":{"n":5000,"median":1538.0},"cost":{"n":5000,"median":0.078675}},"all":{"n":20000,"correct":18691,"accuracy":93.455,"tokens":{"n":20000,"median":567.0},"cost":{"n":20000,"median":0.03051}}}},{"id":"sol-xhigh","label":"GPT-6 Sol","short":"Sol","provider":"openai","scopes":{"tinanta":{"n":5000,"correct":3964,"accuracy":79.28,"tokens":{"n":5000,"median":1821.5},"cost":{"n":5000,"median":0.018668999999999998}},"krdanta":{"n":5000,"correct":4113,"accuracy":82.26,"tokens":{"n":5000,"median":1414.0},"cost":{"n":5000,"median":0.014613000000000001}},"subanta":{"n":5000,"correct":4898,"accuracy":97.96,"tokens":{"n":5000,"median":97.0},"cost":{"n":5000,"median":0.0013}},"taddhitanta":{"n":5000,"correct":3956,"accuracy":79.12,"tokens":{"n":5000,"median":2431.0},"cost":{"n":5000,"median":0.024668000000000002}},"all":{"n":20000,"correct":16931,"accuracy":84.655,"tokens":{"n":20000,"median":1053.0},"cost":{"n":20000,"median":0.010992}}}},{"id":"luna-xhigh","label":"GPT-6 Luna","short":"Luna","provider":"openai","scopes":{"tinanta":{"n":5000,"correct":2584,"accuracy":51.68,"tokens":{"n":5000,"median":4266.0},"cost":{"n":5000,"median":0.0021559500000000002}},"krdanta":{"n":5000,"correct":2939,"accuracy":58.78,"tokens":{"n":5000,"median":2712.0},"cost":{"n":5000,"median":0.0013790999999999999}},"subanta":{"n":5000,"correct":4787,"accuracy":95.74,"tokens":{"n":5000,"median":133.0},"cost":{"n":5000,"median":8.3e-05}},"taddhitanta":{"n":5000,"correct":3079,"accuracy":61.58,"tokens":{"n":5000,"median":6506.0},"cost":{"n":5000,"median":0.0032706000000000002}},"all":{"n":20000,"correct":13389,"accuracy":66.945,"tokens":{"n":20000,"median":2273.5},"cost":{"n":20000,"median":0.0011585}}}},{"id":"opus55-xhigh","label":"Claude Opus 5.5","short":"Opus 5.5","provider":"anthropic","scopes":{"tinanta":{"n":5000,"correct":4890,"accuracy":97.8,"tokens":{"n":5000,"median":354.0},"cost":{"n":5000,"median":0.00845}},"krdanta":{"n":5000,"correct":4908,"accuracy":98.16,"tokens":{"n":5000,"median":293.0},"cost":{"n":5000,"median":0.007288}},"subanta":{"n":5000,"correct":4933,"accuracy":98.66,"tokens":{"n":5000,"median":58.0},"cost":{"n":5000,"median":0.002176}},"taddhitanta":{"n":5000,"correct":4417,"accuracy":88.34,"tokens":{"n":5000,"median":374.0},"cost":{"n":5000,"median":0.008588}},"all":{"n":20000,"correct":19148,"accuracy":95.74,"tokens":{"n":20000,"median":259.0},"cost":{"n":20000,"median":0.006474000000000001}}}},{"id":"sonnet55-xhigh","label":"Claude Sonnet 5.5","short":"Sonnet 5.5","provider":"anthropic","scopes":{"tinanta":{"n":5000,"correct":4201,"accuracy":84.02,"tokens":{"n":5000,"median":974.5},"cost":{"n":5000,"median":0.010425}},"krdanta":{"n":5000,"correct":4361,"accuracy":87.22,"tokens":{"n":5000,"median":756.5},"cost":{"n":5000,"median":0.008279}},"subanta":{"n":5000,"correct":4889,"accuracy":97.78,"tokens":{"n":5000,"median":120.0},"cost":{"n":5000,"median":0.001706}},"taddhitanta":{"n":5000,"correct":4181,"accuracy":83.62,"tokens":{"n":5000,"median":1174.0},"cost":{"n":5000,"median":0.012292}},"all":{"n":20000,"correct":17632,"accuracy":88.16,"tokens":{"n":20000,"median":714.0},"cost":{"n":20000,"median":0.0078}}}},{"id":"haiku55-xhigh","label":"Claude Haiku 5.5","short":"Haiku 5.5","provider":"anthropic","scopes":{"tinanta":{"n":5000,"correct":2107,"accuracy":42.14,"tokens":{"n":5000,"median":3322.0},"cost":{"n":5000,"median":0.0016952999999999998}},"krdanta":{"n":5000,"correct":2645,"accuracy":52.9,"tokens":{"n":5000,"median":2932.5},"cost":{"n":5000,"median":0.0015021000000000001}},"subanta":{"n":5000,"correct":4792,"accuracy":95.84,"tokens":{"n":5000,"median":201.0},"cost":{"n":5000,"median":0.000126}},"taddhitanta":{"n":5000,"correct":2697,"accuracy":53.94,"tokens":{"n":5000,"median":3575.5},"cost":{"n":5000,"median":0.0018153000000000002}},"all":{"n":20000,"correct":12241,"accuracy":61.205,"tokens":{"n":20000,"median":2293.5},"cost":{"n":20000,"median":0.0011796}}}},{"id":"gemini4-argon-high","label":"Gemini 4 Argon","short":"Argon","provider":"google","scopes":{"tinanta":{"n":5000,"correct":4708,"accuracy":94.16,"tokens":{"n":5000,"median":1596.0},"cost":{"n":5000,"median":0.032936}},"krdanta":{"n":5000,"correct":4850,"accuracy":97.0,"tokens":{"n":5000,"median":1050.5},"cost":{"n":5000,"median":0.022058}},"subanta":{"n":5000,"correct":4935,"accuracy":98.7,"tokens":{"n":5000,"median":303.0},"cost":{"n":5000,"median":0.006836}},"taddhitanta":{"n":5000,"correct":4570,"accuracy":91.4,"tokens":{"n":5000,"median":1161.5},"cost":{"n":5000,"median":0.024059999999999998}},"all":{"n":20000,"correct":19063,"accuracy":95.315,"tokens":{"n":20000,"median":942.0},"cost":{"n":20000,"median":0.019814}}}},{"id":"gemini38-flash-high","label":"Gemini 3.8 Flash","short":"Flash","provider":"google","scopes":{"tinanta":{"n":5000,"correct":4511,"accuracy":90.22,"tokens":{"n":5000,"median":9587.5},"cost":{"n":5000,"median":0.07236675000000001}},"krdanta":{"n":5000,"correct":4690,"accuracy":93.8,"tokens":{"n":5000,"median":4956.5},"cost":{"n":5000,"median":0.0375195}},"subanta":{"n":5000,"correct":4916,"accuracy":98.32,"tokens":{"n":5000,"median":383.0},"cost":{"n":5000,"median":0.00312375}},"taddhitanta":{"n":5000,"correct":4359,"accuracy":87.18,"tokens":{"n":5000,"median":10185.0},"cost":{"n":5000,"median":0.07668449999999999}},"all":{"n":20000,"correct":18476,"accuracy":92.38,"tokens":{"n":20000,"median":4369.5},"cost":{"n":20000,"median":0.033092250000000004}}}},{"id":"sgram-14b-answerp1-2304","label":"SGram-14B","short":"SGram-14B","provider":"local","scopes":{"tinanta":{"n":5000,"correct":4682,"accuracy":93.64,"tokens":{"n":5000,"median":16.0},"cost":{"n":0,"median":null}},"krdanta":{"n":5000,"correct":4863,"accuracy":97.26,"tokens":{"n":5000,"median":14.0},"cost":{"n":0,"median":null}},"subanta":{"n":5000,"correct":4938,"accuracy":98.76,"tokens":{"n":5000,"median":17.0},"cost":{"n":0,"median":null}},"taddhitanta":{"n":5000,"correct":4835,"accuracy":96.7,"tokens":{"n":5000,"median":17.0},"cost":{"n":0,"median":null}},"all":{"n":20000,"correct":19318,"accuracy":96.59,"tokens":{"n":20000,"median":16.0},"cost":{"n":0,"median":null}}}}],"paired":{"tinanta":{"astra-xhigh":{"bothCorrect":4473,"sgramOnly":209,"otherOnly":206,"bothWrong":112,"n":5000},"sol-xhigh":{"bothCorrect":3818,"sgramOnly":864,"otherOnly":146,"bothWrong":172,"n":5000},"luna-xhigh":{"bothCorrect":2490,"sgramOnly":2192,"otherOnly":94,"bothWrong":224,"n":5000},"opus55-xhigh":{"bothCorrect":4617,"sgramOnly":65,"otherOnly":273,"bothWrong":45,"n":5000},"sonnet55-xhigh":{"bothCorrect":4032,"sgramOnly":650,"otherOnly":169,"bothWrong":149,"n":5000},"haiku55-xhigh":{"bothCorrect":2010,"sgramOnly":2672,"otherOnly":97,"bothWrong":221,"n":5000},"gemini4-argon-high":{"bothCorrect":4464,"sgramOnly":218,"otherOnly":244,"bothWrong":74,"n":5000},"gemini38-flash-high":{"bothCorrect":4320,"sgramOnly":362,"otherOnly":191,"bothWrong":127,"n":5000}},"krdanta":{"astra-xhigh":{"bothCorrect":4722,"sgramOnly":141,"otherOnly":84,"bothWrong":53,"n":5000},"sol-xhigh":{"bothCorrect":4037,"sgramOnly":826,"otherOnly":76,"bothWrong":61,"n":5000},"luna-xhigh":{"bothCorrect":2884,"sgramOnly":1979,"otherOnly":55,"bothWrong":82,"n":5000},"opus55-xhigh":{"bothCorrect":4791,"sgramOnly":72,"otherOnly":117,"bothWrong":20,"n":5000},"sonnet55-xhigh":{"bothCorrect":4284,"sgramOnly":579,"otherOnly":77,"bothWrong":60,"n":5000},"haiku55-xhigh":{"bothCorrect":2589,"sgramOnly":2274,"otherOnly":56,"bothWrong":81,"n":5000},"gemini4-argon-high":{"bothCorrect":4743,"sgramOnly":120,"otherOnly":107,"bothWrong":30,"n":5000},"gemini38-flash-high":{"bothCorrect":4593,"sgramOnly":270,"otherOnly":97,"bothWrong":40,"n":5000}},"subanta":{"astra-xhigh":{"bothCorrect":4917,"sgramOnly":21,"otherOnly":9,"bothWrong":53,"n":5000},"sol-xhigh":{"bothCorrect":4890,"sgramOnly":48,"otherOnly":8,"bothWrong":54,"n":5000},"luna-xhigh":{"bothCorrect":4780,"sgramOnly":158,"otherOnly":7,"bothWrong":55,"n":5000},"opus55-xhigh":{"bothCorrect":4924,"sgramOnly":14,"otherOnly":9,"bothWrong":53,"n":5000},"sonnet55-xhigh":{"bothCorrect":4879,"sgramOnly":59,"otherOnly":10,"bothWrong":52,"n":5000},"haiku55-xhigh":{"bothCorrect":4785,"sgramOnly":153,"otherOnly":7,"bothWrong":55,"n":5000},"gemini4-argon-high":{"bothCorrect":4926,"sgramOnly":12,"otherOnly":9,"bothWrong":53,"n":5000},"gemini38-flash-high":{"bothCorrect":4908,"sgramOnly":30,"otherOnly":8,"bothWrong":54,"n":5000}},"taddhitanta":{"astra-xhigh":{"bothCorrect":4271,"sgramOnly":564,"otherOnly":9,"bothWrong":156,"n":5000},"sol-xhigh":{"bothCorrect":3952,"sgramOnly":883,"otherOnly":4,"bothWrong":161,"n":5000},"luna-xhigh":{"bothCorrect":3077,"sgramOnly":1758,"otherOnly":2,"bothWrong":163,"n":5000},"opus55-xhigh":{"bothCorrect":4414,"sgramOnly":421,"otherOnly":3,"bothWrong":162,"n":5000},"sonnet55-xhigh":{"bothCorrect":4177,"sgramOnly":658,"otherOnly":4,"bothWrong":161,"n":5000},"haiku55-xhigh":{"bothCorrect":2695,"sgramOnly":2140,"otherOnly":2,"bothWrong":163,"n":5000},"gemini4-argon-high":{"bothCorrect":4557,"sgramOnly":278,"otherOnly":13,"bothWrong":152,"n":5000},"gemini38-flash-high":{"bothCorrect":4352,"sgramOnly":483,"otherOnly":7,"bothWrong":158,"n":5000}},"all":{"astra-xhigh":{"bothCorrect":18383,"sgramOnly":935,"otherOnly":308,"bothWrong":374,"n":20000},"sol-xhigh":{"bothCorrect":16697,"sgramOnly":2621,"otherOnly":234,"bothWrong":448,"n":20000},"luna-xhigh":{"bothCorrect":13231,"sgramOnly":6087,"otherOnly":158,"bothWrong":524,"n":20000},"opus55-xhigh":{"bothCorrect":18746,"sgramOnly":572,"otherOnly":402,"bothWrong":280,"n":20000},"sonnet55-xhigh":{"bothCorrect":17372,"sgramOnly":1946,"otherOnly":260,"bothWrong":422,"n":20000},"haiku55-xhigh":{"bothCorrect":12079,"sgramOnly":7239,"otherOnly":162,"bothWrong":520,"n":20000},"gemini4-argon-high":{"bothCorrect":18690,"sgramOnly":628,"otherOnly":373,"bothWrong":309,"n":20000},"gemini38-flash-high":{"bothCorrect":18173,"sgramOnly":1145,"otherOnly":303,"bothWrong":379,"n":20000}}},"consensus":{"tinanta":[{"n":48,"sgramCorrect":8},{"n":27,"sgramCorrect":13},{"n":75,"sgramCorrect":45},{"n":106,"sgramCorrect":75},{"n":283,"sgramCorrect":255},{"n":718,"sgramCorrect":677},{"n":1223,"sgramCorrect":1176},{"n":1071,"sgramCorrect":1029},{"n":1449,"sgramCorrect":1404}],"krdanta":[{"n":15,"sgramCorrect":7},{"n":26,"sgramCorrect":16},{"n":71,"sgramCorrect":56},{"n":86,"sgramCorrect":72},{"n":195,"sgramCorrect":185},{"n":571,"sgramCorrect":560},{"n":958,"sgramCorrect":941},{"n":1121,"sgramCorrect":1095},{"n":1957,"sgramCorrect":1931}],"subanta":[{"n":61,"sgramCorrect":9},{"n":2,"sgramCorrect":1},{"n":4,"sgramCorrect":4},{"n":3,"sgramCorrect":3},{"n":2,"sgramCorrect":2},{"n":19,"sgramCorrect":18},{"n":69,"sgramCorrect":69},{"n":180,"sgramCorrect":177},{"n":4660,"sgramCorrect":4655}],"taddhitanta":[{"n":203,"sgramCorrect":57},{"n":138,"sgramCorrect":127},{"n":104,"sgramCorrect":101},{"n":120,"sgramCorrect":119},{"n":139,"sgramCorrect":137},{"n":476,"sgramCorrect":476},{"n":762,"sgramCorrect":762},{"n":1139,"sgramCorrect":1139},{"n":1919,"sgramCorrect":1917}],"all":[{"n":327,"sgramCorrect":81},{"n":193,"sgramCorrect":157},{"n":254,"sgramCorrect":206},{"n":315,"sgramCorrect":269},{"n":619,"sgramCorrect":579},{"n":1784,"sgramCorrect":1731},{"n":3012,"sgramCorrect":2948},{"n":3511,"sgramCorrect":3440},{"n":9985,"sgramCorrect":9907}]},"sft":{"answer-p1":{"step":2304,"tokens":4719193,"results":{"krdanta":{"tasks":5000,"samples":80000,"pass_at_1":0.973825,"pass_at_16":0.979,"format_compliance":1.0,"answer_membership":0.973825,"mean_output_tokens":14.0346125,"truncation_rate":0.0,"first_sample_accuracy":0.9726,"first_sample_success":0.9726},"subanta":{"tasks":5000,"samples":80000,"pass_at_1":0.9878375,"pass_at_16":0.9886,"format_compliance":1.0,"answer_membership":0.9878375,"mean_output_tokens":17.09075,"truncation_rate":0.0,"first_sample_accuracy":0.9876,"first_sample_success":0.9876},"taddhitanta":{"tasks":5000,"samples":80000,"pass_at_1":0.9668,"pass_at_16":0.9678,"format_compliance":1.0,"answer_membership":0.9668,"mean_output_tokens":16.8705875,"truncation_rate":0.0,"first_sample_accuracy":0.967,"first_sample_success":0.967},"tinanta":{"tasks":5000,"samples":80000,"pass_at_1":0.9360375,"pass_at_16":0.947,"format_compliance":1.0,"answer_membership":0.9360375,"mean_output_tokens":16.4278,"truncation_rate":0.0,"first_sample_accuracy":0.9364,"first_sample_success":0.9364},"macro":{"pass_at_1":0.966125,"pass_at_16":0.9706,"format_compliance":1.0,"answer_membership":0.966125,"mean_output_tokens":16.1059375,"truncation_rate":0.0,"first_sample_accuracy":0.9659,"first_sample_success":0.9659}}},"answer-p16":{"step":1280,"tokens":2621651,"results":{"krdanta":{"tasks":5000,"samples":80000,"pass_at_1":0.9659875,"pass_at_16":0.9776,"format_compliance":1.0,"answer_membership":0.9659875,"mean_output_tokens":14.03875,"truncation_rate":0.0,"first_sample_accuracy":0.9656,"first_sample_success":0.9656},"subanta":{"tasks":5000,"samples":80000,"pass_at_1":0.983225,"pass_at_16":0.987,"format_compliance":1.0,"answer_membership":0.983225,"mean_output_tokens":17.0894625,"truncation_rate":0.0,"first_sample_accuracy":0.9832,"first_sample_success":0.9832},"taddhitanta":{"tasks":5000,"samples":80000,"pass_at_1":0.9665625,"pass_at_16":0.9828,"format_compliance":1.0,"answer_membership":0.9665625,"mean_output_tokens":16.8629875,"truncation_rate":0.0,"first_sample_accuracy":0.9666,"first_sample_success":0.9666},"tinanta":{"tasks":5000,"samples":80000,"pass_at_1":0.9191875,"pass_at_16":0.9392,"format_compliance":1.0,"answer_membership":0.9191875,"mean_output_tokens":16.4408,"truncation_rate":0.0,"first_sample_accuracy":0.922,"first_sample_success":0.922},"macro":{"pass_at_1":0.958740625,"pass_at_16":0.97165,"format_compliance":1.0,"answer_membership":0.958740625,"mean_output_tokens":16.108,"truncation_rate":0.0,"first_sample_accuracy":0.95935,"first_sample_success":0.95935}}},"derivation-p1":{"step":2176,"tokens":4458023,"results":{"krdanta":{"tasks":5000,"samples":80000,"pass_at_1":0.8589875,"pass_at_16":0.95,"format_compliance":1.0,"answer_membership":0.8589875,"mean_output_tokens":146.0280125,"truncation_rate":0.0,"first_sample_accuracy":0.8564,"first_sample_success":0.8564},"subanta":{"tasks":5000,"samples":80000,"pass_at_1":0.93385,"pass_at_16":0.9842,"format_compliance":1.0,"answer_membership":0.93385,"mean_output_tokens":89.1584375,"truncation_rate":0.0,"first_sample_accuracy":0.9366,"first_sample_success":0.9366},"taddhitanta":{"tasks":5000,"samples":80000,"pass_at_1":0.9126125,"pass_at_16":0.9762,"format_compliance":0.9999875,"answer_membership":0.9126125,"mean_output_tokens":111.5133625,"truncation_rate":1.25e-05,"first_sample_accuracy":0.917,"first_sample_success":0.917},"tinanta":{"tasks":5000,"samples":80000,"pass_at_1":0.7255375,"pass_at_16":0.9154,"format_compliance":1.0,"answer_membership":0.7255375,"mean_output_tokens":153.9210375,"truncation_rate":0.0,"first_sample_accuracy":0.733,"first_sample_success":0.733},"macro":{"pass_at_1":0.857746875,"pass_at_16":0.95645,"format_compliance":0.999996875,"answer_membership":0.857746875,"mean_output_tokens":125.1552125,"truncation_rate":3.125e-06,"first_sample_accuracy":0.86075,"first_sample_success":0.86075}}},"derivation-p16":{"step":2304,"tokens":4719971,"results":{"krdanta":{"tasks":5000,"samples":80000,"pass_at_1":0.856475,"pass_at_16":0.953,"format_compliance":1.0,"answer_membership":0.856475,"mean_output_tokens":148.683025,"truncation_rate":0.0,"first_sample_accuracy":0.852,"first_sample_success":0.852},"subanta":{"tasks":5000,"samples":80000,"pass_at_1":0.9351,"pass_at_16":0.9846,"format_compliance":1.0,"answer_membership":0.9351,"mean_output_tokens":88.873875,"truncation_rate":0.0,"first_sample_accuracy":0.9374,"first_sample_success":0.9374},"taddhitanta":{"tasks":5000,"samples":80000,"pass_at_1":0.9029625,"pass_at_16":0.9816,"format_compliance":1.0,"answer_membership":0.9029625,"mean_output_tokens":111.511175,"truncation_rate":0.0,"first_sample_accuracy":0.9034,"first_sample_success":0.9034},"tinanta":{"tasks":5000,"samples":80000,"pass_at_1":0.728225,"pass_at_16":0.9196,"format_compliance":1.0,"answer_membership":0.728225,"mean_output_tokens":153.8862875,"truncation_rate":0.0,"first_sample_accuracy":0.7362,"first_sample_success":0.7362},"macro":{"pass_at_1":0.855690625,"pass_at_16":0.9597,"format_compliance":1.0,"answer_membership":0.855690625,"mean_output_tokens":125.738590625,"truncation_rate":0.0,"first_sample_accuracy":0.85725,"first_sample_success":0.85725}}}},"sampling":{"tinanta":{"n":5000,"never":265,"always":4623,"points":[{"k":1,"accuracy":93.60374999999999},{"k":2,"accuracy":93.98166666666667},{"k":4,"accuracy":94.28583516483516},{"k":8,"accuracy":94.52931313131313},{"k":16,"accuracy":94.69999999999999}]},"krdanta":{"n":5000,"never":105,"always":4842,"points":[{"k":1,"accuracy":97.38250000000001},{"k":2,"accuracy":97.56233333333333},{"k":4,"accuracy":97.69891208791209},{"k":8,"accuracy":97.80353846153847},{"k":16,"accuracy":97.89999999999999}]},"subanta":{"n":5000,"never":57,"always":4934,"points":[{"k":1,"accuracy":98.78375},{"k":2,"accuracy":98.81383333333333},{"k":4,"accuracy":98.83423076923077},{"k":8,"accuracy":98.84793006993007},{"k":16,"accuracy":98.86}]},"taddhitanta":{"n":5000,"never":161,"always":4828,"points":[{"k":1,"accuracy":96.67999999999999},{"k":2,"accuracy":96.71416666666667},{"k":4,"accuracy":96.74096703296703},{"k":8,"accuracy":96.76307381507382},{"k":16,"accuracy":96.78}]},"all":{"n":20000,"never":588,"always":19227,"points":[{"k":1,"accuracy":96.6125},{"k":2,"accuracy":96.768},{"k":4,"accuracy":96.88998626373626},{"k":8,"accuracy":96.98596386946386},{"k":16,"accuracy":97.06}]}},"uncertainty":[{"checkpoint":"answer-p1","step":2304,"frontier_id":"astra-xhigh","frontier_label":"GPT-6 Astra","sft_first_sample_percent":96.59,"frontier_percent":93.455,"difference_pp":3.1350000000000002,"sft_only_correct":935,"frontier_only_correct":308,"exact_mcnemar":{"p":9.05916588314105e-74,"log10_p":-73.04291178786684},"bonferroni_32_p":2.898933082605136e-72,"paired_bootstrap_95_ci_pp":[2.8000000000000003,3.4749999999999996],"cluster_sensitivity":{"clusters":7538,"delta_pp":3.135,"standard_error_pp":0.20035988123230428,"wald_95_ci_pp":[2.7423018488379602,3.5276981511620384],"normal_approximation_p":3.4910306817877696e-55,"bonferroni_32_p":1.1171298181720863e-53},"secondary_mean16_difference_pp":3.1574999999999998,"secondary_mean16_bootstrap_95_ci_pp":[2.8246875,3.4915624999999997],"per_category_difference_pp":{"krdanta":1.1400000000000001,"subanta":0.24,"taddhitanta":11.1,"tinanta":0.06},"holm_8_log10_p":-72.56579053314718,"holm_8_p":2.7177497649424213e-73},{"checkpoint":"answer-p1","step":2304,"frontier_id":"sol-xhigh","frontier_label":"GPT-6 Sol","sft_first_sample_percent":96.59,"frontier_percent":84.655,"difference_pp":11.935,"sft_only_correct":2621,"frontier_only_correct":234,"exact_mcnemar":{"p":0.0,"log10_p":-509.10743386861725},"bonferroni_32_p":0.0,"paired_bootstrap_95_ci_pp":[11.450000000000001,12.42],"cluster_sensitivity":{"clusters":7538,"delta_pp":11.935,"standard_error_pp":0.33744738452203255,"wald_95_ci_pp":[11.273615279659577,12.596384720340422],"normal_approximation_p":5.213903580498507e-274,"bonferroni_32_p":1.668449145759522e-272},"secondary_mean16_difference_pp":11.9575,"secondary_mean16_bootstrap_95_ci_pp":[11.472187500000002,12.449695312500001],"per_category_difference_pp":{"krdanta":15.0,"subanta":0.8,"taddhitanta":17.580000000000002,"tinanta":14.360000000000001},"holm_8_log10_p":-508.3292826182336,"holm_8_p":0.0},{"checkpoint":"answer-p1","step":2304,"frontier_id":"luna-xhigh","frontier_label":"GPT-6 Luna","sft_first_sample_percent":96.59,"frontier_percent":66.945,"difference_pp":29.645,"sft_only_correct":6087,"frontier_only_correct":158,"exact_mcnemar":{"p":0.0,"log10_p":-1561.0635708764341},"bonferroni_32_p":0.0,"paired_bootstrap_95_ci_pp":[29.025000000000002,30.264999999999997],"cluster_sensitivity":{"clusters":7538,"delta_pp":29.645,"standard_error_pp":0.457761689282796,"wald_95_ci_pp":[28.747803575503504,30.542196424496492],"normal_approximation_p":0.0,"bonferroni_32_p":0.0},"secondary_mean16_difference_pp":29.667500000000004,"secondary_mean16_bootstrap_95_ci_pp":[29.05125,30.2878125],"per_category_difference_pp":{"krdanta":38.48,"subanta":3.02,"taddhitanta":35.120000000000005,"tinanta":41.959999999999994},"holm_8_log10_p":-1560.2184728364198,"holm_8_p":0.0},{"checkpoint":"answer-p1","step":2304,"frontier_id":"opus55-xhigh","frontier_label":"Claude Opus 5.5","sft_first_sample_percent":96.59,"frontier_percent":95.74000000000001,"difference_pp":0.8500000000000001,"sft_only_correct":572,"frontier_only_correct":402,"exact_mcnemar":{"p":5.6868962419998434e-08,"log10_p":-7.245124695422533},"bonferroni_32_p":1.8198067974399499e-06,"paired_bootstrap_95_ci_pp":[0.5549999999999997,1.15],"cluster_sensitivity":{"clusters":7538,"delta_pp":0.8500000000000001,"standard_error_pp":0.1885155904059158,"wald_95_ci_pp":[0.4805162322801005,1.2194837677198997],"normal_approximation_p":6.516126644257798e-06,"bonferroni_32_p":0.00020851605261624953},"secondary_mean16_difference_pp":0.8725,"secondary_mean16_bootstrap_95_ci_pp":[0.5824921874999998,1.1675000000000002],"per_category_difference_pp":{"krdanta":-0.8999999999999999,"subanta":0.1,"taddhitanta":8.36,"tinanta":-4.16},"holm_8_log10_p":-7.245124695422533,"holm_8_p":5.6868962419997865e-08},{"checkpoint":"answer-p1","step":2304,"frontier_id":"sonnet55-xhigh","frontier_label":"Claude Sonnet 5.5","sft_first_sample_percent":96.59,"frontier_percent":88.16000000000001,"difference_pp":8.43,"sft_only_correct":1946,"frontier_only_correct":260,"exact_mcnemar":{"p":1.38134e-318,"log10_p":-317.8596994222782},"bonferroni_32_p":4.420283e-317,"paired_bootstrap_95_ci_pp":[7.994999999999999,8.875],"cluster_sensitivity":{"clusters":7538,"delta_pp":8.43,"standard_error_pp":0.2991132988824001,"wald_95_ci_pp":[7.84374870689353,9.01625129310647],"normal_approximation_p":9.368944122557543e-175,"bonferroni_32_p":2.9980621192184137e-173},"secondary_mean16_difference_pp":8.4525,"secondary_mean16_bootstrap_95_ci_pp":[8.016250000000001,8.8921875],"per_category_difference_pp":{"krdanta":10.040000000000001,"subanta":0.98,"taddhitanta":13.08,"tinanta":9.62},"holm_8_log10_p":-317.16072941794215,"holm_8_p":6.9067e-318},{"checkpoint":"answer-p1","step":2304,"frontier_id":"haiku55-xhigh","frontier_label":"Claude Haiku 5.5","sft_first_sample_percent":96.59,"frontier_percent":61.205,"difference_pp":35.385,"sft_only_correct":7239,"frontier_only_correct":162,"exact_mcnemar":{"p":0.0,"log10_p":-1890.6477733955198},"bonferroni_32_p":0.0,"paired_bootstrap_95_ci_pp":[34.755,36.015],"cluster_sensitivity":{"clusters":7538,"delta_pp":35.385,"standard_error_pp":0.45175155846334525,"wald_95_ci_pp":[34.499583215452,36.270416784548],"normal_approximation_p":0.0,"bonferroni_32_p":0.0},"secondary_mean16_difference_pp":35.4075,"secondary_mean16_bootstrap_95_ci_pp":[34.778125,36.035624999999996],"per_category_difference_pp":{"krdanta":44.36,"subanta":2.92,"taddhitanta":42.76,"tinanta":51.5},"holm_8_log10_p":-1889.7446834085279,"holm_8_p":0.0},{"checkpoint":"answer-p1","step":2304,"frontier_id":"gemini4-argon-high","frontier_label":"Gemini 4 Argon","sft_first_sample_percent":96.59,"frontier_percent":95.31500000000001,"difference_pp":1.275,"sft_only_correct":628,"frontier_only_correct":373,"exact_mcnemar":{"p":6.938483625497404e-16,"log10_p":-15.15873543228804},"bonferroni_32_p":2.2203147601591692e-14,"paired_bootstrap_95_ci_pp":[0.9649999999999999,1.58],"cluster_sensitivity":{"clusters":7538,"delta_pp":1.2750000000000001,"standard_error_pp":0.19710470587009835,"wald_95_ci_pp":[0.8886818753112468,1.6613181246887536],"normal_approximation_p":9.888683017477683e-11,"bonferroni_32_p":3.1643785655928584e-09},"secondary_mean16_difference_pp":1.2975,"secondary_mean16_bootstrap_95_ci_pp":[0.9971875000000001,1.5962499999999997],"per_category_difference_pp":{"krdanta":0.26,"subanta":0.06,"taddhitanta":5.3,"tinanta":-0.52},"holm_8_log10_p":-14.857705436624059,"holm_8_p":1.3876967250994604e-15},{"checkpoint":"answer-p1","step":2304,"frontier_id":"gemini38-flash-high","frontier_label":"Gemini 3.8 Flash","sft_first_sample_percent":96.59,"frontier_percent":92.38,"difference_pp":4.21,"sft_only_correct":1145,"frontier_only_correct":303,"exact_mcnemar":{"p":3.445431905237308e-115,"log10_p":-114.46275632896567},"bonferroni_32_p":1.1025382096759386e-113,"paired_bootstrap_95_ci_pp":[3.8449999999999998,4.575],"cluster_sensitivity":{"clusters":7538,"delta_pp":4.210000000000001,"standard_error_pp":0.2489289018446429,"wald_95_ci_pp":[3.7221083176733942,4.697891682326607],"normal_approximation_p":3.641853529750868e-64,"bonferroni_32_p":1.1653931295202778e-62},"secondary_mean16_difference_pp":4.2325,"secondary_mean16_bootstrap_95_ci_pp":[3.8696875,4.5953124999999995],"per_category_difference_pp":{"krdanta":3.46,"subanta":0.44,"taddhitanta":9.520000000000001,"tinanta":3.42},"holm_8_log10_p":-113.8606963376377,"holm_8_p":1.3781727620948317e-114}],"methodology":{"metric":"Completed exact-answer membership; SGram fixed sample 0 unless labeled pass@k.","efficiency":"Median per-trial usage/cost across all selected tasks, including failures. Provider-reported output tokens include reasoning where reported; no double counting. Recorded selected-result input+output cost, before credits; historical retry charges are not reconstructed.","missing":"Incomplete cost/token measurements are omitted, never zero-filled. SGram has no measured per-task API cost.","comparability":"Prompts, decoding, tokenizers, and model/provider pricing differ. Cross-model association is descriptive, not causal.","sources_sha256":{"data/benchmark/tinanta/tinanta.json":"e5f1b8b6332d820ef3ffd11d28e73c25df300c253816ad374f716fb2b537d151","data/benchmark/krdanta/krdanta.json":"ec5fcaa3ee7d05ff79d391d0015b26e896bd01873a720cc94bff26bf5970e762","data/benchmark/subanta/subanta.json":"83bc5976715a52f6b09535d7a8a50a629072e2e6e19fdf1281c682126a994d53","data/benchmark/taddhitanta/taddhitanta.json":"ae3f1084be43b8296fe7a5cc3c1264ea84ea1293fc72c0644b02ce9e6f036c3b","data/benchmark/results/astra-xhigh/tinanta.json":"088817bc65ff4a72954c3eb514e43a92af4557279c233eb2f09236dd2d604742","data/benchmark/results/astra-xhigh/krdanta.json":"88b5b0483efaf599ee97c995d7266090ea10610159f2cb61368c4550861255b8","data/benchmark/results/astra-xhigh/subanta.json":"485bd570d684572d562004a89377b7dc904c8a1322f1161fca54d4e7e0f12573","data/benchmark/results/astra-xhigh/taddhitanta.json":"78ea9d0328a4aabf64cc4ab8100cb6293485ee23fd38e20631134f1503e87ea5","data/benchmark/results/sol-xhigh/tinanta.json":"78f157b2c99f040c0f471ee191b41c6d3b3be6d4c33077a12905776ccbd9ce9b","data/benchmark/results/sol-xhigh/krdanta.json":"261c099b378eb3589faac3debd5ef0c3faa4decb0f7937e910b3e7b0c8abb80d","data/benchmark/results/sol-xhigh/subanta.json":"363af85cb737aa4f53816e4aee6ae5fb7f16c8db8b1d48bcea1210e232f4f8b3","data/benchmark/results/sol-xhigh/taddhitanta.json":"b83f05a97b2f2d5b2dd7f7fc61c62d1531decc06dfef331da7646002489cec7b","data/benchmark/results/luna-xhigh/tinanta.json":"5886291a63917dede8c26f1720c909885688a4c71d64438c937f5a5dd3ba4357","data/benchmark/results/luna-xhigh/krdanta.json":"a10862222b0d8f40748dcb20896e210f073a535646f6b7c9d55358c1f7f87e7a","data/benchmark/results/luna-xhigh/subanta.json":"5f99b1e92aa4aacdf01044a0c2e23cace0ac46fb7364f388811fb29aa1d3b8ba","data/benchmark/results/luna-xhigh/taddhitanta.json":"bc47987b13df8010b9419d079aa83eb7b15719599f3ebe34fcf7f0af87c77f10","data/benchmark/results/opus55-xhigh/tinanta.json":"963bdd3ee999c6ccafa346a5d44352cc1f0a2707c345a10f08608d18fdd2ed56","data/benchmark/results/opus55-xhigh/krdanta.json":"8b057a53227c1ecae02dc717c3a1ba6c7eb52699c99954c5e263f672b3074af9","data/benchmark/results/opus55-xhigh/subanta.json":"0d2e5ef42a2fb088d81654d85cebc03e10cccccb49936fabf1b6780e360275ee","data/benchmark/results/opus55-xhigh/taddhitanta.json":"891d37f41691c29732184c1a7554c680d316243e1df752c3a2f698d6ad6d4958","data/benchmark/results/sonnet55-xhigh/tinanta.json":"45553659876dc75e250c9ea3274227046b1aca999878d0524bf477c577861dec","data/benchmark/results/sonnet55-xhigh/krdanta.json":"8a87afd56115fd9e0b0b0dbce6572f79fb3f4e2aacafb5e0f4fc8b67e3f9fd30","data/benchmark/results/sonnet55-xhigh/subanta.json":"bb631df4156cfed0805ff3596e17a71abaeaff4e3909692fd13b4ef907378de8","data/benchmark/results/sonnet55-xhigh/taddhitanta.json":"f984da050920a3c3d2a9380b0c90f2ba8521004bc04e6eae5264eb64eabd4cb2","data/benchmark/results/haiku55-xhigh/tinanta.json":"2ebce90101b5ed10238776c2bb1722278b51503d79dbcfa4f79811c6bbc59d38","data/benchmark/results/haiku55-xhigh/krdanta.json":"9cef284f6f64f47d596918d88e6105badbd3560f0033b2c19411ee527d5d3f1b","data/benchmark/results/haiku55-xhigh/subanta.json":"ceec2433d30e5c3391eb7b5ff816d237667a57cd6e9a395235f0dcba07c0257e","data/benchmark/results/haiku55-xhigh/taddhitanta.json":"61892b9fcb56da5be698ad5a28123ae462318f7c1f4ec0f0f874cd05a3181524","data/benchmark/results/gemini4-argon-high/tinanta.json":"b87cf52d37410c15017a4e5c39ba6e6b7a1ef3f05548b0da8e43b1c24b833f55","data/benchmark/results/gemini4-argon-high/krdanta.json":"23804236e790ad64156c91ec9d93faad5f2efecb7e7be26e541f61661e99164e","data/benchmark/results/gemini4-argon-high/subanta.json":"5420ef8397b55df4264cb46594b7e71c7f92338f2447606197123ddab2d62f05","data/benchmark/results/gemini4-argon-high/taddhitanta.json":"3e7e170494e0f8b7b152df710aceda87bb8add9d7cb7d2d164f744ed6bd939c2","data/benchmark/results/gemini38-flash-high/tinanta.json":"311a47f29191c2df9ed30a09af7124d12155e57bb9d4ca563e615ae6f451303b","data/benchmark/results/gemini38-flash-high/krdanta.json":"e7b4c3d5409cf007bf868d89c49a7e1f167955925686cbefc74bafdf61526ee6","data/benchmark/results/gemini38-flash-high/subanta.json":"554f84815d546762ec80e5209f4c7a68bb54d37e886b9fbfbb189b681b487e7c","data/benchmark/results/gemini38-flash-high/taddhitanta.json":"8c9ac6828bea85fecd692a3de45c5cf8b3419783d1161aab9da6dd311ffb0570","data/benchmark/results/sgram-14b-answerp1-2304/tinanta.json":"d45f04e7342a67fa61d0c091eddbcd5b00a32815aefc94d1e4d222f96572c065","data/benchmark/results/sgram-14b-answerp1-2304/krdanta.json":"b8b7348efee74756b1b2dc01a933b72306e0ba8ee112c7aff3ab3c6a08b7dfbd","data/benchmark/results/sgram-14b-answerp1-2304/subanta.json":"04ae3945940d15d19586611495c789455aceb9929a4ab576b9ebeb495188a81d","data/benchmark/results/sgram-14b-answerp1-2304/taddhitanta.json":"aed5b5e24cc56109ffa64f26e1d9beec796975797a6435037109004d9183ac18","data/benchmark/results/sft-selected-20261019/answer-p1/summary.json":"3d535441c8cc6b39b2c6ed125b4479697c94de238c07de33fd5bfdf71edc011c","data/benchmark/results/sft-selected-20261019/answer-p16/summary.json":"2ddf740db95e3bbbb567f8905d306bb2785a4a47fa68bc99f4ecadecbdf268b7","data/benchmark/results/sft-selected-20261019/derivation-p1/summary.json":"bf5db230c408cdeb2739b9fae772e3423916ea64f3048c6f0cd7e2722818e0b9","data/benchmark/results/sft-selected-20261019/derivation-p16/summary.json":"c1cb91b8c5a6e24c24023c12e96bc9ca4bb39e0e9cc9ed378cba89e943a0aa79","data/benchmark/results/sft-selected-20261019/significance/answer-p1.task-scores.json":"faff00fa754da5878025ad32f83e021233f1612b11b1deac39162a084ad72d9b","data/benchmark/results/sft-selected-20261019/significance/report.json":"ad6cf394f49572d570b3ae1851d6aa463bb66b20d40cd2193feb428723ac9831"}}}