{ "metadata": { "benchmark": "Vals Index", "slug": "vals_index", "description": "A single measure of AI's potential economic impact \u2014 agentic model performance across finance, coding, legal, and tax tasks, weighted by each sector's share of U.S. GDP.", "benchmark_id": "vals_index", "family": "vals_index", "version": "2.2", "updated": "2026-10-27", "dataset_type": "private", "industry": "index", "tasks": { "overall": "Overall", "finance_agent": "Finance Agent v2.1", "emb": "EMB v1.1", "terminal_bench_4": "Terminal-Bench 4.0", "vibe_code_bench": "Vibe Code Bench v1.1", "code_migration": "Code Migration v1.1", "legal_research": "Legal Research", "legal_agent_benchmark": "HLAB-Vals-1.1", "tax_agent_bench": "Tax Agent" }, "models": [ "alibaba/qwen3.8-max-internal", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash-1017", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.7", "inception/mercury-3", "kimi/kimi-k3", "kimi/kimi-k3.1-pro", "meta/muse-spark-open", "meta/muse-wonder", "meta/muse-wonder-full", "minimax/MiniMax-M3.1-flash", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6.1-sol", "openai/gpt-6.1-terra", "reflection/Beam-1.0", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 28 }, "tasks": { "overall": { "anthropic/claude-sonnet-5-5": { "accuracy": 70.854, "latency": 5158.766, "stderr": 0.928, "cost_per_test": 18.69955, "avg_input_tokens": 77990774.9, "avg_output_tokens": 594549.6, "avg_reasoning_tokens": 405075.1, "avg_cache_read_tokens": 76487919.7, "avg_cache_write_tokens": 1502345.7, "token_totals": { "input_tokens": 45087854954, "output_tokens": 654942865, "reasoning_tokens": 510663474, "cache_read_tokens": 43320518175, "cache_write_tokens": 1766865965 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 70.177, "latency": 5071.089, "stderr": 1.004, "cost_per_test": 28.122421, "avg_input_tokens": 59002489.2, "avg_output_tokens": 455039.7, "avg_reasoning_tokens": 293955.7, "avg_cache_read_tokens": 57785923.7, "avg_cache_write_tokens": 1216167.3, "token_totals": { "input_tokens": 31937474844, "output_tokens": 496373071, "reasoning_tokens": 378363763, "cache_read_tokens": 30652283666, "cache_write_tokens": 1284864516 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 69.237, "latency": 2697.953, "stderr": 1.09, "cost_per_test": 13.372731, "avg_input_tokens": 23148371.5, "avg_output_tokens": 159229.4, "avg_reasoning_tokens": 58309.9, "avg_cache_read_tokens": 21661266.3, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 13360828360, "output_tokens": 146167290, "reasoning_tokens": 59887853, "cache_read_tokens": 12071919973, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 66.787, "latency": 3558.574, "stderr": 1.066, "cost_per_test": 6.809627, "avg_input_tokens": 18271139.5, "avg_output_tokens": 103432.9, "avg_reasoning_tokens": 48524.7, "avg_cache_read_tokens": 17923954.3, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 11051421288, "output_tokens": 96958718, "reasoning_tokens": 50663639, "cache_read_tokens": 10611361430, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-astra": { "accuracy": 65.023, "latency": 2018.879, "stderr": 1.098, "cost_per_test": 21.277828, "avg_input_tokens": 8045900.6, "avg_output_tokens": 99101.8, "avg_reasoning_tokens": 51905.9, "avg_cache_read_tokens": 7645164.4, "token_totals": { "input_tokens": 4924948729, "output_tokens": 96766878, "reasoning_tokens": 59928445, "cache_read_tokens": 4484842803, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 64.877, "latency": 2123.575, "stderr": 1.067, "cost_per_test": 10.647504, "avg_input_tokens": 33073266.1, "avg_output_tokens": 115811.6, "avg_reasoning_tokens": 59021.7, "avg_cache_read_tokens": 32525940.1, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 18224226557, "output_tokens": 97195877, "reasoning_tokens": 52653929, "cache_read_tokens": 17645554776, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 63.944, "latency": 2664.03, "stderr": 1.072, "cost_per_test": 3.614054, "avg_input_tokens": 9291986.4, "avg_output_tokens": 112502.2, "avg_reasoning_tokens": 62769.0, "avg_cache_read_tokens": 8988645.9, "avg_cache_write_tokens": 302569.0, "token_totals": { "input_tokens": 5486504345, "output_tokens": 103203979, "reasoning_tokens": 64580497, "cache_read_tokens": 5078686427, "cache_write_tokens": 406706672 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 62.52, "latency": 3816.452, "stderr": 1.141, "cost_per_test": 6.440192, "avg_input_tokens": 19980586.0, "avg_output_tokens": 159071.7, "avg_reasoning_tokens": 96642.2, "avg_cache_read_tokens": 19358617.6, "avg_cache_write_tokens": 612758.3, "token_totals": { "input_tokens": 12819249449, "output_tokens": 188000009, "reasoning_tokens": 133071106, "cache_read_tokens": 12002927488, "cache_write_tokens": 808823104 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 61.257, "latency": 2394.709, "stderr": 1.055, "cost_per_test": 0.65747, "avg_input_tokens": 43644461.6, "avg_output_tokens": 258045.4, "avg_reasoning_tokens": 174021.7, "avg_cache_read_tokens": 43300368.5, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 27106524900, "output_tokens": 324131672, "reasoning_tokens": 251666265, "cache_read_tokens": 26546819188, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 60.168, "latency": 2381.407, "stderr": 1.03, "cost_per_test": 2.924759, "avg_input_tokens": 17278227.6, "avg_output_tokens": 157492.9, "avg_reasoning_tokens": 110552.9, "avg_cache_read_tokens": 16794931.9, "avg_cache_write_tokens": 482194.5, "token_totals": { "input_tokens": 10374501981, "output_tokens": 221380500, "reasoning_tokens": 179634864, "cache_read_tokens": 9627236433, "cache_write_tokens": 745894559 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 59.729, "latency": 3864.087, "stderr": 1.111, "cost_per_test": 8.524625, "avg_input_tokens": 27949602.7, "avg_output_tokens": 202266.1, "avg_cache_read_tokens": 27410401.9, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 15044468726, "output_tokens": 216344074, "reasoning_tokens": null, "cache_read_tokens": 14478264797, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 59.045, "latency": 6345.62, "stderr": 1.082, "cost_per_test": 8.729516, "avg_input_tokens": 26354758.4, "avg_output_tokens": 231723.9, "avg_reasoning_tokens": 155010.9, "avg_cache_read_tokens": 25797460.1, "token_totals": { "input_tokens": 16155618982, "output_tokens": 266182756, "reasoning_tokens": 201430787, "cache_read_tokens": 15384763435, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 58.727, "latency": 2996.841, "stderr": 1.282, "cost_per_test": 3.17475, "avg_input_tokens": 37579232.6, "avg_output_tokens": 570127.1, "avg_reasoning_tokens": 459156.5, "avg_cache_read_tokens": 36601039.8, "avg_cache_write_tokens": 977877.9, "token_totals": { "input_tokens": 23100726075, "output_tokens": 772700077, "reasoning_tokens": 679539502, "cache_read_tokens": 21634995270, "cache_write_tokens": 1465402149 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 58.564, "latency": 2013.837, "stderr": 1.013, "cost_per_test": 11.625468, "avg_input_tokens": 11268193.4, "avg_output_tokens": 124146.2, "avg_reasoning_tokens": 68843.5, "avg_cache_read_tokens": 10616594.6, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 7301646245, "output_tokens": 112664299, "reasoning_tokens": 65804651, "cache_read_tokens": 6541333803, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 57.813, "latency": 4355.892, "stderr": 1.24, "cost_per_test": 0.42958, "avg_input_tokens": 17623254.7, "avg_output_tokens": 147276.8, "avg_reasoning_tokens": 95076.7, "avg_cache_read_tokens": 17072238.0, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 9588782585, "output_tokens": 115944789, "reasoning_tokens": 76759715, "cache_read_tokens": 9117346901, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 55.951, "latency": 2158.482, "stderr": 1.141, "cost_per_test": 5.025617, "avg_input_tokens": 24930668.6, "avg_output_tokens": 130206.4, "avg_reasoning_tokens": 68436.7, "avg_cache_read_tokens": 23839395.3, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 15367814865, "output_tokens": 117528442, "reasoning_tokens": 68777600, "cache_read_tokens": 14301116094, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 54.826, "latency": 4027.975, "stderr": 1.114, "cost_per_test": 0.221007, "avg_input_tokens": 26689642.9, "avg_output_tokens": 172166.6, "avg_reasoning_tokens": 106440.1, "avg_cache_read_tokens": 25928380.5, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 14236355547, "output_tokens": 128500987, "reasoning_tokens": 79445287, "cache_read_tokens": 13652865579, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 53.781, "latency": 2597.835, "stderr": 1.067, "cost_per_test": 0.659684, "avg_input_tokens": 31517784.7, "avg_output_tokens": 255284.6, "avg_reasoning_tokens": 183512.9, "avg_cache_read_tokens": 31161890.7, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 18180722777, "output_tokens": 306409526, "reasoning_tokens": 243120444, "cache_read_tokens": 17662797762, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "tencent/hy4-preview": { "accuracy": 53.209, "latency": 3562.636, "stderr": 1.13, "cost_per_test": 1.56824, "avg_input_tokens": 19074387.8, "avg_output_tokens": 160621.0, "avg_reasoning_tokens": 105106.1, "avg_cache_read_tokens": 18555324.8, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 10732575937, "output_tokens": 186491684, "reasoning_tokens": 140958004, "cache_read_tokens": 10205230838, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 52.974, "latency": 3482.913, "stderr": 1.044, "cost_per_test": 2.499691, "avg_input_tokens": 43178249.2, "avg_output_tokens": 169366.1, "avg_reasoning_tokens": 97785.0, "avg_cache_read_tokens": 42932509.5, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 22172718464, "output_tokens": 154910470, "reasoning_tokens": 106624381, "cache_read_tokens": 21810722517, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 52.955, "latency": 4496.011, "stderr": 1.001, "cost_per_test": 7.169465, "avg_input_tokens": 15196000.1, "avg_output_tokens": 107341.6, "avg_reasoning_tokens": 58602.9, "avg_cache_read_tokens": 14785760.0, "avg_cache_write_tokens": 285912.5, "token_totals": { "input_tokens": 8947298492, "output_tokens": 88481723, "reasoning_tokens": 52290753, "cache_read_tokens": 8512028610, "cache_write_tokens": 312874126 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 52.322, "latency": 1943.192, "stderr": 1.117, "cost_per_test": 0.450147, "avg_input_tokens": 17348582.2, "avg_output_tokens": 170502.4, "avg_reasoning_tokens": 124778.7, "avg_cache_read_tokens": 16946836.9, "avg_cache_write_tokens": 400386.5, "token_totals": { "input_tokens": 10050409316, "output_tokens": 201660024, "reasoning_tokens": 167309610, "cache_read_tokens": 9523623079, "cache_write_tokens": 525274447 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 50.861, "latency": 3830.634, "stderr": 1.255, "cost_per_test": 1.133938, "avg_input_tokens": 62213903.2, "avg_output_tokens": 203646.5, "avg_cache_read_tokens": 60767985.3, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 31476965553, "output_tokens": 178936231, "reasoning_tokens": null, "cache_read_tokens": 30533831295, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 48.567, "latency": 4925.409, "stderr": 1.087, "cost_per_test": 4.71792, "avg_input_tokens": 9735429.3, "avg_output_tokens": 65357.2, "avg_reasoning_tokens": 33474.7, "avg_cache_read_tokens": 6991662.4, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 10323622551, "output_tokens": 61830033, "reasoning_tokens": 31607549, "cache_read_tokens": 7631312123, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "inception/mercury-3": { "accuracy": 43.659, "latency": 1325.888, "stderr": 1.227, "cost_per_test": 0.477863, "avg_input_tokens": 20204701.2, "avg_output_tokens": 117494.0, "avg_reasoning_tokens": 105632.2, "avg_cache_read_tokens": 17311926.8, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 10777503004, "output_tokens": 108097979, "reasoning_tokens": 96647515, "cache_read_tokens": 9066624996, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "reflection/Beam-1.0": { "accuracy": 43.424, "latency": 446.562, "stderr": 1.062, "cost_per_test": null, "avg_input_tokens": 5912885.2, "avg_output_tokens": 41553.0, "avg_reasoning_tokens": 22329.7, "avg_cache_read_tokens": 5673911.8, "token_totals": { "input_tokens": 4260716050, "output_tokens": 44440148, "reasoning_tokens": 26896514, "cache_read_tokens": 3868334848, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling": { "accuracy": 28.961, "latency": 1032.925, "stderr": 1.12, "cost_per_test": 1.847767, "avg_input_tokens": 7482988.3, "avg_output_tokens": 64607.2, "avg_cache_read_tokens": 7298094.8, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 5136674424, "output_tokens": 73075439, "reasoning_tokens": null, "cache_read_tokens": 4844827008, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 27.588, "latency": 986.441, "stderr": 1.133, "cost_per_test": 0.401307, "avg_input_tokens": 5328516.9, "avg_output_tokens": 55978.2, "avg_cache_read_tokens": 5206791.8, "avg_cache_write_tokens": 0.0, "token_totals": { "input_tokens": 3504374025, "output_tokens": 65026929, "reasoning_tokens": null, "cache_read_tokens": 3321415595, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "vibe_code_bench": { "anthropic/claude-sonnet-5-5": { "accuracy": 92.392, "latency": 3672.106, "stderr": 1.26, "cost_per_test": 31.250578, "token_totals": { "input_tokens": 5822927217, "output_tokens": 25491041, "reasoning_tokens": 11893204, "cache_read_tokens": 5760733058, "cache_write_tokens": 62167153 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 219.9, "avg_query_seconds": 16.7, "median_query_seconds": 7.8, "avg_ttft_seconds": 11.7, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "google/gemini-4-argon": { "accuracy": 91.906, "latency": 2247.675, "stderr": 1.899, "cost_per_test": 8.10797, "token_totals": { "input_tokens": 451864686, "output_tokens": 5436370, "reasoning_tokens": 849287, "cache_read_tokens": 397575696, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 72.8, "avg_query_seconds": 30.9, "median_query_seconds": 13.9, "avg_ttft_seconds": 15.0, "output_time_split": { "reasoning_pct": 82.5, "content_pct": 0.0, "tool_call_pct": 17.5 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "anthropic/claude-haiku-5-5": { "accuracy": 90.444, "latency": 2075.723, "stderr": 1.725, "cost_per_test": 6.071856, "token_totals": { "input_tokens": 4600663825, "output_tokens": 23863970, "reasoning_tokens": 13235182, "cache_read_tokens": 4559642564, "cache_write_tokens": 40996929 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 205.2, "avg_query_seconds": 10.1, "median_query_seconds": 4.6, "avg_ttft_seconds": 7.1, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "anthropic/claude-opus-5-5": { "accuracy": 90.294, "latency": 5766.524, "stderr": 1.526, "cost_per_test": 57.922811, "token_totals": { "input_tokens": 5596583425, "output_tokens": 19722501, "reasoning_tokens": 7457724, "cache_read_tokens": 5539450465, "cache_write_tokens": 57102136 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 231.0, "avg_query_seconds": 25.0, "median_query_seconds": 11.6, "avg_ttft_seconds": 18.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "openai/gpt-6-astra": { "accuracy": 89.594, "latency": 2344.127, "stderr": 2.174, "cost_per_test": 38.512323, "token_totals": { "input_tokens": 813962842, "output_tokens": 7672087, "reasoning_tokens": 3013580, "cache_read_tokens": 762745769, "cache_write_tokens": null }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 106.1, "avg_query_seconds": 22.1, "median_query_seconds": 9.7, "avg_ttft_seconds": 11.5, "output_time_split": { "reasoning_pct": 8.0, "content_pct": 0.7, "tool_call_pct": 91.2 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "openai/gpt-6.1-sol": { "accuracy": 88.931, "latency": 4489.755, "stderr": 1.893, "cost_per_test": 6.234794, "token_totals": { "input_tokens": 1073502468, "output_tokens": 10186788, "reasoning_tokens": 5383848, "cache_read_tokens": 1056860976, "cache_write_tokens": 16623336 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 121.0, "avg_query_seconds": 37.1, "median_query_seconds": 12.0, "avg_ttft_seconds": 26.4, "output_time_split": { "reasoning_pct": 6.5, "content_pct": 1.2, "tool_call_pct": 92.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 87.73, "latency": 973.834, "stderr": 2.752, "cost_per_test": 0.579984, "token_totals": { "input_tokens": 2173326299, "output_tokens": 9246207, "reasoning_tokens": 3107112, "cache_read_tokens": 2156782720, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 170.2, "avg_query_seconds": 5.7, "median_query_seconds": 3.9, "avg_ttft_seconds": 2.2, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "grok/grok-4.7": { "accuracy": 86.175, "latency": 2147.065, "stderr": 2.181, "cost_per_test": 15.82706, "token_totals": { "input_tokens": 767137889, "output_tokens": 7923865, "reasoning_tokens": 3838088, "cache_read_tokens": 713123456, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 86.4, "avg_query_seconds": 24.8, "median_query_seconds": 8.5, "avg_ttft_seconds": 3.4, "output_time_split": { "reasoning_pct": 75.7, "content_pct": 4.2, "tool_call_pct": 20.2 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "meta/muse-wonder-full": { "accuracy": 85.996, "latency": 648.336, "stderr": 1.933, "cost_per_test": 2.293448, "token_totals": { "input_tokens": 222902077, "output_tokens": 2071146, "reasoning_tokens": 144351, "cache_read_tokens": 215357588, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 60.5, "avg_query_seconds": 10.7, "median_query_seconds": 5.5, "avg_ttft_seconds": 6.6, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "xiaomi/mimo-v2.6-pro": { "accuracy": 85.223, "latency": 3646.574, "stderr": 3.392, "cost_per_test": 1.043846, "token_totals": { "input_tokens": 1076595168, "output_tokens": 7478091, "reasoning_tokens": 2852217, "cache_read_tokens": 979676672, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 111.4, "avg_query_seconds": 32.7, "median_query_seconds": 18.1, "avg_ttft_seconds": 13.8, "output_time_split": { "reasoning_pct": 36.4, "content_pct": 0.7, "tool_call_pct": 62.9 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands" }, "kimi/kimi-k3": { "accuracy": 84.968, "latency": 999.021, "stderr": 2.746, "cost_per_test": 10.007576, "token_totals": { "input_tokens": 1261633418, "output_tokens": 5010801, "reasoning_tokens": 1248736, "cache_read_tokens": 1244327217, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 139.6, "avg_query_seconds": 7.2, "median_query_seconds": 6.2, "avg_ttft_seconds": 4.4, "output_time_split": { "reasoning_pct": 33.5, "content_pct": 2.5, "tool_call_pct": 64.1 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "OpenHands" }, "kimi/kimi-k3.1-pro": { "accuracy": 84.894, "latency": 3028.937, "stderr": 2.949, "cost_per_test": 7.485151, "token_totals": { "input_tokens": 1401107533, "output_tokens": 5806937, "reasoning_tokens": 1661626, "cache_read_tokens": 1381126038, "cache_write_tokens": 19409526 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 131.4, "avg_query_seconds": 23.1, "median_query_seconds": 14.0, "avg_ttft_seconds": 10.5, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "OpenHands" }, "openai/gpt-6.1-terra": { "accuracy": 83.997, "latency": 2250.002, "stderr": 2.865, "cost_per_test": 5.996953, "token_totals": { "input_tokens": 2454324986, "output_tokens": 5928692, "reasoning_tokens": 1847456, "cache_read_tokens": 2424165406, "cache_write_tokens": 30133951 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 170.9, "avg_query_seconds": 13.2, "median_query_seconds": 9.4, "avg_ttft_seconds": 7.0, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "meta/muse-wonder": { "accuracy": 83.089, "latency": 350.076, "stderr": 2.49, "cost_per_test": 1.802723, "token_totals": { "input_tokens": 144518172, "output_tokens": 1830347, "reasoning_tokens": 114242, "cache_read_tokens": 138037760, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 52.3, "avg_query_seconds": 6.7, "median_query_seconds": 3.6, "avg_ttft_seconds": 4.3, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 82.297, "latency": 3982.6, "stderr": 3.148, "cost_per_test": 0.356023, "token_totals": { "input_tokens": 1830028483, "output_tokens": 7746431, "reasoning_tokens": 2336919, "cache_read_tokens": 1820296320, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "openai/gpt-6-luna": { "accuracy": 81.649, "latency": 2219.515, "stderr": 3.377, "cost_per_test": 1.345703, "token_totals": { "input_tokens": 3186713261, "output_tokens": 9004550, "reasoning_tokens": 3632878, "cache_read_tokens": 3158117507, "cache_write_tokens": 28566327 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 196.2, "avg_query_seconds": 11.3, "median_query_seconds": 7.6, "avg_ttft_seconds": 5.5, "output_time_split": { "reasoning_pct": 28.4, "content_pct": 0.0, "tool_call_pct": 71.6 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "xiaomi/mimo-v2.6-flash": { "accuracy": 78.96, "latency": 3013.012, "stderr": 4.045, "cost_per_test": 0.563041, "token_totals": { "input_tokens": 1311689878, "output_tokens": 7947370, "reasoning_tokens": 3093850, "cache_read_tokens": 1157695808, "cache_write_tokens": 0 }, "token_tasks_measured": 48, "turn_metrics": { "tasks_measured": 48, "avg_turns": 122.0, "avg_query_seconds": 24.7, "median_query_seconds": 14.8, "avg_ttft_seconds": 13.5, "output_time_split": { "reasoning_pct": 32.7, "content_pct": 0.6, "tool_call_pct": 66.7 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands" }, "google/gemini-3.8-flash": { "accuracy": 78.651, "latency": 519.104, "stderr": 3.884, "cost_per_test": 6.865141, "token_totals": { "input_tokens": 1149521706, "output_tokens": 6144190, "reasoning_tokens": 1439332, "cache_read_tokens": 1057116240, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 146.1, "avg_query_seconds": 3.6, "median_query_seconds": 2.8, "avg_ttft_seconds": 2.6, "output_time_split": { "reasoning_pct": 96.1, "content_pct": 0.0, "tool_call_pct": 3.9 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "zai/glm-5.3": { "accuracy": 78.125, "latency": 3847.197, "stderr": 3.94, "cost_per_test": 12.450185, "token_totals": { "input_tokens": 1971471587, "output_tokens": 8173329, "reasoning_tokens": null, "cache_read_tokens": 1906590911, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 194.4, "avg_query_seconds": 19.8, "median_query_seconds": 8.0, "avg_ttft_seconds": 11.9, "output_time_split": { "reasoning_pct": 42.9, "content_pct": 1.1, "tool_call_pct": 56.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "tencent/hy4-preview": { "accuracy": 77.478, "latency": 2493.104, "stderr": 4.035, "cost_per_test": 2.181516, "token_totals": { "input_tokens": 1648623995, "output_tokens": 6263219, "reasoning_tokens": 1671679, "cache_read_tokens": 1618107200, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 178.5, "avg_query_seconds": 14.0, "median_query_seconds": 8.9, "avg_ttft_seconds": 5.2, "output_time_split": { "reasoning_pct": 31.8, "content_pct": 0.8, "tool_call_pct": 67.4 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": "OpenHands" }, "alibaba/qwen3.8-max-internal": { "accuracy": 77.141, "latency": 5079.378, "stderr": 3.721, "cost_per_test": 11.668329, "token_totals": { "input_tokens": 1895650724, "output_tokens": 8327720, "reasoning_tokens": 2363284, "cache_read_tokens": 1861629312, "cache_write_tokens": null }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 171.6, "avg_query_seconds": 29.6, "median_query_seconds": 18.6, "avg_ttft_seconds": 17.6, "output_time_split": { "reasoning_pct": 33.2, "content_pct": 0.3, "tool_call_pct": 66.5 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "meta/muse-spark-open": { "accuracy": 76.187, "latency": 1869.172, "stderr": 3.317, "cost_per_test": 1.655609, "token_totals": { "input_tokens": 225685852, "output_tokens": 2085879, "reasoning_tokens": 952456, "cache_read_tokens": 189265312, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 70.3, "avg_query_seconds": 26.6, "median_query_seconds": 6.8, "avg_ttft_seconds": 6.7, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": "OpenHands" }, "minimax/MiniMax-M3.1-flash": { "accuracy": 73.944, "latency": 1263.916, "stderr": 4.249, "cost_per_test": 1.146649, "token_totals": { "input_tokens": 3334725307, "output_tokens": 8488142, "reasoning_tokens": 2462335, "cache_read_tokens": 3318290676, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 231.6, "avg_query_seconds": 5.5, "median_query_seconds": 4.5, "avg_ttft_seconds": 3.0, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": "OpenHands" }, "reflection/Beam-1.0": { "accuracy": 66.436, "latency": 145.263, "stderr": 4.062, "cost_per_test": null, "token_totals": { "input_tokens": 430528251, "output_tokens": 2004844, "reasoning_tokens": 550281, "cache_read_tokens": 419321600, "cache_write_tokens": null }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 129.5, "avg_query_seconds": 1.1, "median_query_seconds": 0.6, "avg_ttft_seconds": 0.5, "output_time_split": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": "OpenHands" }, "inception/mercury-3": { "accuracy": 60.927, "latency": 566.58, "stderr": 5.379, "cost_per_test": 0.627191, "token_totals": { "input_tokens": 1422007111, "output_tokens": 6493005, "reasoning_tokens": 5135055, "cache_read_tokens": 1256819130, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 171.6, "avg_query_seconds": 3.3, "median_query_seconds": 2.2, "avg_ttft_seconds": 2.5, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": "OpenHands" }, "zai/glm-5.3-flash": { "accuracy": 30.759, "latency": 5027.281, "stderr": 5.277, "cost_per_test": 2.348805, "token_totals": { "input_tokens": 7308851703, "output_tokens": 11916569, "reasoning_tokens": null, "cache_read_tokens": 7228379520, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "turn_metrics": { "tasks_measured": 50, "avg_turns": 556.0, "avg_query_seconds": 9.0, "median_query_seconds": 5.5, "avg_ttft_seconds": 3.1, "output_time_split": { "reasoning_pct": 37.2, "content_pct": 1.7, "tool_call_pct": 61.1 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "thinkingmachines/inkling": { "accuracy": 19.214, "latency": 1105.492, "stderr": 3.659, "cost_per_test": 1.605949, "token_totals": { "input_tokens": 391977485, "output_tokens": 2288277, "reasoning_tokens": 0, "cache_read_tokens": 386683776, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": "OpenHands" }, "thinkingmachines/inkling-small": { "accuracy": 19.052, "latency": 2949.469, "stderr": 3.403, "cost_per_test": 0.432264, "token_totals": { "input_tokens": 294407624, "output_tokens": 2135670, "reasoning_tokens": null, "cache_read_tokens": 289327616, "cache_write_tokens": 0 }, "token_tasks_measured": 50, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": "OpenHands" } }, "finance_agent": { "anthropic/claude-opus-5-5": { "accuracy": 67.846, "latency": 1980.122, "stderr": 2.051, "cost_per_test": 8.908721, "token_totals": { "input_tokens": 711290297, "output_tokens": 106934250, "reasoning_tokens": 92568283, "cache_read_tokens": 362746415, "cache_write_tokens": 348503177 }, "token_tasks_measured": 444, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 67.658, "latency": 793.03, "stderr": 2.0, "cost_per_test": 4.315759, "token_totals": { "input_tokens": 602790494, "output_tokens": 31627905, "reasoning_tokens": 14149611, "cache_read_tokens": 291037965, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 67.575, "latency": 850.298, "stderr": 1.98, "cost_per_test": 2.680843, "token_totals": { "input_tokens": 389317179, "output_tokens": 18193553, "reasoning_tokens": 11613004, "cache_read_tokens": 188800282, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 67.275, "latency": 2066.221, "stderr": 2.023, "cost_per_test": 6.950246, "token_totals": { "input_tokens": 1206037903, "output_tokens": 148942679, "reasoning_tokens": 130899975, "cache_read_tokens": 579490734, "cache_write_tokens": 626479999 }, "token_tasks_measured": 448, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 65.761, "latency": 443.888, "stderr": 2.039, "cost_per_test": 1.991077, "token_totals": { "input_tokens": 1102307399, "output_tokens": 26345690, "reasoning_tokens": 18599088, "cache_read_tokens": 708933383, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 65.545, "latency": 310.129, "stderr": 2.031, "cost_per_test": 2.620705, "token_totals": { "input_tokens": 398379578, "output_tokens": 15303665, "reasoning_tokens": 9870222, "cache_read_tokens": 190235588, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 65.373, "latency": 1864.634, "stderr": 2.063, "cost_per_test": 1.637243, "token_totals": { "input_tokens": 1065081554, "output_tokens": 206334140, "reasoning_tokens": 194190119, "cache_read_tokens": 378369304, "cache_write_tokens": 686667014 }, "token_tasks_measured": 449, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 65.004, "latency": 1205.572, "stderr": 2.01, "cost_per_test": 1.030939, "token_totals": { "input_tokens": 323459972, "output_tokens": 37030623, "reasoning_tokens": 30619069, "cache_read_tokens": 134111615, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 64.259, "latency": 682.01, "stderr": 2.015, "cost_per_test": 0.183321, "token_totals": { "input_tokens": 292706558, "output_tokens": 11189430, "reasoning_tokens": 6591988, "cache_read_tokens": 126914944, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 63.316, "latency": 936.347, "stderr": 2.084, "cost_per_test": 0.066534, "token_totals": { "input_tokens": 389007728, "output_tokens": 14596133, "reasoning_tokens": 8588760, "cache_read_tokens": 208996480, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 62.933, "latency": 909.212, "stderr": 2.028, "cost_per_test": 0.466261, "token_totals": { "input_tokens": 1025601275, "output_tokens": 94944123, "reasoning_tokens": 82978352, "cache_read_tokens": 721979776, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 61.817, "latency": 1145.078, "stderr": 2.112, "cost_per_test": 1.626543, "token_totals": { "input_tokens": 253535763, "output_tokens": 21994277, "reasoning_tokens": 16385214, "cache_read_tokens": 70737183, "cache_write_tokens": 182747233 }, "token_tasks_measured": 449, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 61.623, "latency": 1502.748, "stderr": 2.059, "cost_per_test": 0.667449, "token_totals": { "input_tokens": 332001046, "output_tokens": 49959222, "reasoning_tokens": 42800144, "cache_read_tokens": 128979992, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 61.488, "latency": 2099.046, "stderr": 2.1, "cost_per_test": 2.290605, "token_totals": { "input_tokens": 430824171, "output_tokens": 49084271, "reasoning_tokens": 40045316, "cache_read_tokens": 180005120, "cache_write_tokens": 249767698 }, "token_tasks_measured": 449, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 60.665, "latency": 1612.347, "stderr": 2.056, "cost_per_test": 2.758047, "token_totals": { "input_tokens": 3043998798, "output_tokens": 14268191, "reasoning_tokens": 7831933, "cache_read_tokens": 2388431856, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 60.587, "latency": 296.095, "stderr": 2.032, "cost_per_test": 0.072473, "token_totals": { "input_tokens": 231343920, "output_tokens": 17576955, "reasoning_tokens": 13137649, "cache_read_tokens": 91246328, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.7": { "accuracy": 60.585, "latency": 652.292, "stderr": 2.114, "cost_per_test": 2.238652, "token_totals": { "input_tokens": 741416449, "output_tokens": 22618555, "reasoning_tokens": 15034963, "cache_read_tokens": 461563904, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 60.428, "latency": 1228.75, "stderr": 2.12, "cost_per_test": 7.823464, "token_totals": { "input_tokens": 248231715, "output_tokens": 22170755, "reasoning_tokens": 16637610, "cache_read_tokens": 77618056, "cache_write_tokens": 170564517 }, "token_tasks_measured": 449, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 60.013, "latency": 875.039, "stderr": 2.09, "cost_per_test": 1.881233, "token_totals": { "input_tokens": 400595105, "output_tokens": 15119518, "reasoning_tokens": 9960408, "cache_read_tokens": 216957696, "cache_write_tokens": 182544978 }, "token_tasks_measured": 448, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 58.829, "latency": 3319.592, "stderr": 2.117, "cost_per_test": 2.992239, "token_totals": { "input_tokens": 910481004, "output_tokens": 72516574, "reasoning_tokens": 60279740, "cache_read_tokens": 521454848, "cache_write_tokens": null }, "token_tasks_measured": 449, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 58.826, "latency": 904.204, "stderr": 2.07, "cost_per_test": 0.158978, "token_totals": { "input_tokens": 469331836, "output_tokens": 54339785, "reasoning_tokens": 45403914, "cache_read_tokens": 234241552, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 57.116, "latency": 476.985, "stderr": 2.083, "cost_per_test": 0.82234, "token_totals": { "input_tokens": 380103725, "output_tokens": 27027936, "reasoning_tokens": 21919502, "cache_read_tokens": 187724800, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 56.841, "latency": 926.859, "stderr": 2.123, "cost_per_test": 0.120919, "token_totals": { "input_tokens": 393427873, "output_tokens": 45914348, "reasoning_tokens": 42042763, "cache_read_tokens": 189468255, "cache_write_tokens": 203907655 }, "token_tasks_measured": 449, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 55.958, "latency": 1787.129, "stderr": 2.094, "cost_per_test": 1.922848, "token_totals": { "input_tokens": 723048978, "output_tokens": 69759698, "reasoning_tokens": 60468509, "cache_read_tokens": 356407138, "cache_write_tokens": 366491932 }, "token_tasks_measured": 449, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-3": { "accuracy": 54.708, "latency": 232.758, "stderr": 1.991, "cost_per_test": 0.064622, "token_totals": { "input_tokens": 338680752, "output_tokens": 21159893, "reasoning_tokens": 17980953, "cache_read_tokens": 136207667, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "reflection/Beam-1.0": { "accuracy": 51.827, "latency": 85.425, "stderr": 2.107, "cost_per_test": null, "token_totals": { "input_tokens": 503284975, "output_tokens": 15066321, "reasoning_tokens": 11184585, "cache_read_tokens": 255618816, "cache_write_tokens": null }, "token_tasks_measured": 449, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling": { "accuracy": 49.873, "latency": 573.747, "stderr": 2.116, "cost_per_test": 0.891309, "token_totals": { "input_tokens": 845699789, "output_tokens": 26758592, "reasoning_tokens": null, "cache_read_tokens": 667318528, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 44.776, "latency": 314.216, "stderr": 2.095, "cost_per_test": 0.207921, "token_totals": { "input_tokens": 641180329, "output_tokens": 23499273, "reasoning_tokens": null, "cache_read_tokens": 529986304, "cache_write_tokens": 0 }, "token_tasks_measured": 449, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "emb": { "anthropic/claude-sonnet-5-5": { "accuracy": 75.899, "latency": 2204.991, "stderr": 2.468, "cost_per_test": 8.792863, "token_totals": { "input_tokens": 2367951142, "output_tokens": 29328538, "reasoning_tokens": 22320203, "cache_read_tokens": 2311427347, "cache_write_tokens": 56507215 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 77.9, "avg_query_seconds": 28.3, "median_query_seconds": 10.8, "avg_ttft_seconds": 24.5, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 75.836, "latency": 2106.434, "stderr": 2.41, "cost_per_test": 11.018974, "token_totals": { "input_tokens": 922243024, "output_tokens": 23704546, "reasoning_tokens": 18569025, "cache_read_tokens": 861165257, "cache_write_tokens": 61067293 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 42.7, "avg_query_seconds": 49.3, "median_query_seconds": 11.8, "avg_ttft_seconds": 33.8, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 74.998, "latency": 1684.504, "stderr": 2.141, "cost_per_test": 4.58223, "token_totals": { "input_tokens": 983252171, "output_tokens": 8330149, "reasoning_tokens": 5099887, "cache_read_tokens": 955848482, "cache_write_tokens": 0 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 44.6, "avg_query_seconds": 37.8, "median_query_seconds": 14.3, "avg_ttft_seconds": 29.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 74.828, "latency": 2305.821, "stderr": 2.494, "cost_per_test": 11.8029, "token_totals": { "input_tokens": 1127460088, "output_tokens": 14257556, "reasoning_tokens": 9497334, "cache_read_tokens": 945025185, "cache_write_tokens": 0 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 42.0, "avg_query_seconds": 54.9, "median_query_seconds": 24.8, "avg_ttft_seconds": 12.8, "output_time_split": { "reasoning_pct": 99.3, "content_pct": 0.0, "tool_call_pct": 0.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 73.33, "latency": 756.889, "stderr": 2.235, "cost_per_test": 0.559258, "token_totals": { "input_tokens": 238952524, "output_tokens": 5755731, "reasoning_tokens": 3532004, "cache_read_tokens": 227336516, "cache_write_tokens": 11372480 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 27.1, "avg_query_seconds": 27.9, "median_query_seconds": 12.4, "avg_ttft_seconds": 5.3, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 72.871, "latency": 769.197, "stderr": 2.424, "cost_per_test": 8.243862, "token_totals": { "input_tokens": 2960728904, "output_tokens": 21722815, "reasoning_tokens": 13162823, "cache_read_tokens": 2787511478, "cache_write_tokens": 0 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 125.0, "avg_query_seconds": 6.1, "median_query_seconds": 3.4, "avg_ttft_seconds": 2.3, "output_time_split": { "reasoning_pct": 98.3, "content_pct": 0.0, "tool_call_pct": 1.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 72.683, "latency": 804.28, "stderr": 2.297, "cost_per_test": 5.827063, "token_totals": { "input_tokens": 123989946, "output_tokens": 5542631, "reasoning_tokens": 3051513, "cache_read_tokens": 104449111, "cache_write_tokens": null }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 17.7, "avg_query_seconds": 45.3, "median_query_seconds": 15.5, "avg_ttft_seconds": 12.1, "output_time_split": { "reasoning_pct": 6.6, "content_pct": 0.9, "tool_call_pct": 92.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 72.503, "latency": 1034.437, "stderr": 2.414, "cost_per_test": 5.599337, "token_totals": { "input_tokens": 1072958990, "output_tokens": 8128857, "reasoning_tokens": 5046443, "cache_read_tokens": 1021915980, "cache_write_tokens": 0 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 47.2, "avg_query_seconds": 21.9, "median_query_seconds": 8.3, "avg_ttft_seconds": 17.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 71.979, "latency": 3193.384, "stderr": 2.387, "cost_per_test": 4.430287, "token_totals": { "input_tokens": 1260695298, "output_tokens": 14484385, "reasoning_tokens": 10207311, "cache_read_tokens": 1232232960, "cache_write_tokens": null }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 68.8, "avg_query_seconds": 46.4, "median_query_seconds": 24.3, "avg_ttft_seconds": 18.8, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.7": { "accuracy": 71.498, "latency": 2097.023, "stderr": 2.328, "cost_per_test": 6.481284, "token_totals": { "input_tokens": 531961928, "output_tokens": 15529729, "reasoning_tokens": 10549516, "cache_read_tokens": 448911360, "cache_write_tokens": 0 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 33.3, "avg_query_seconds": 63.0, "median_query_seconds": 18.7, "avg_ttft_seconds": 5.0, "output_time_split": { "reasoning_pct": 83.7, "content_pct": 14.9, "tool_call_pct": 1.3 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 71.264, "latency": 741.42, "stderr": 2.283, "cost_per_test": 3.074655, "token_totals": { "input_tokens": 410193956, "output_tokens": 9698383, "reasoning_tokens": 7045672, "cache_read_tokens": 393497338, "cache_write_tokens": 0 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 37.6, "avg_query_seconds": 19.7, "median_query_seconds": 10.0, "avg_ttft_seconds": 5.6, "output_time_split": { "reasoning_pct": 80.9, "content_pct": 0.6, "tool_call_pct": 18.5 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 71.088, "latency": 1107.811, "stderr": 2.748, "cost_per_test": 1.143199, "token_totals": { "input_tokens": 129631093, "output_tokens": 6056928, "reasoning_tokens": 3495567, "cache_read_tokens": 112922114, "cache_write_tokens": 16703171 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 17.9, "avg_query_seconds": 61.7, "median_query_seconds": 17.6, "avg_ttft_seconds": 21.8, "output_time_split": { "reasoning_pct": 4.7, "content_pct": 0.9, "tool_call_pct": 94.4 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 70.0, "latency": 1283.322, "stderr": 2.63, "cost_per_test": 1.588749, "token_totals": { "input_tokens": 339381460, "output_tokens": 6447626, "reasoning_tokens": 4047839, "cache_read_tokens": 322881536, "cache_write_tokens": 16153802 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 27.0, "avg_query_seconds": 47.5, "median_query_seconds": 19.7, "avg_ttft_seconds": 7.9, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 69.954, "latency": 1041.052, "stderr": 2.542, "cost_per_test": 0.141708, "token_totals": { "input_tokens": 362285754, "output_tokens": 11007243, "reasoning_tokens": 8590626, "cache_read_tokens": 332678280, "cache_write_tokens": 29598418 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 26.5, "avg_query_seconds": 39.2, "median_query_seconds": 9.5, "avg_ttft_seconds": 4.2, "output_time_split": { "reasoning_pct": 61.3, "content_pct": 0.0, "tool_call_pct": 38.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 69.173, "latency": 1971.687, "stderr": 2.395, "cost_per_test": 0.330922, "token_totals": { "input_tokens": 1809205546, "output_tokens": 16117195, "reasoning_tokens": 11510477, "cache_read_tokens": 1797102837, "cache_write_tokens": 0 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 97.6, "avg_query_seconds": 20.2, "median_query_seconds": 4.9, "avg_ttft_seconds": 1.4, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 69.112, "latency": 2172.231, "stderr": 2.649, "cost_per_test": 3.787827, "token_totals": { "input_tokens": 942177449, "output_tokens": 18177772, "reasoning_tokens": null, "cache_read_tokens": 891631657, "cache_write_tokens": 0 }, "token_tasks_measured": 101, "turn_metrics": { "tasks_measured": 101, "avg_turns": 52.1, "avg_query_seconds": 41.7, "median_query_seconds": 14.5, "avg_ttft_seconds": 4.4, "output_time_split": { "reasoning_pct": 81.2, "content_pct": 0.4, "tool_call_pct": 18.4 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 68.946, "latency": 1839.41, "stderr": 2.402, "cost_per_test": 0.787396, "token_totals": { "input_tokens": 824410363, "output_tokens": 10431429, "reasoning_tokens": 7444891, "cache_read_tokens": 799662720, "cache_write_tokens": 0 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 59.4, "avg_query_seconds": 31.0, "median_query_seconds": 12.6, "avg_ttft_seconds": 4.3, "output_time_split": { "reasoning_pct": 76.3, "content_pct": 0.7, "tool_call_pct": 22.9 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 68.882, "latency": 1521.906, "stderr": 2.267, "cost_per_test": 0.108634, "token_totals": { "input_tokens": 891374421, "output_tokens": 16056047, "reasoning_tokens": 12655956, "cache_read_tokens": 861570496, "cache_write_tokens": 0 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 44.8, "avg_query_seconds": 33.9, "median_query_seconds": 15.2, "avg_ttft_seconds": 5.6, "output_time_split": { "reasoning_pct": 79.3, "content_pct": 0.4, "tool_call_pct": 20.2 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 67.843, "latency": 1847.78, "stderr": 2.29, "cost_per_test": 0.268958, "token_totals": { "input_tokens": 1127000678, "output_tokens": 14186544, "reasoning_tokens": 10606514, "cache_read_tokens": 1112061687, "cache_write_tokens": 0 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 64.0, "avg_query_seconds": 28.9, "median_query_seconds": 6.4, "avg_ttft_seconds": 1.2, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 67.347, "latency": 3313.924, "stderr": 2.903, "cost_per_test": 0.328999, "token_totals": { "input_tokens": 847310790, "output_tokens": 17554862, "reasoning_tokens": 14594001, "cache_read_tokens": 813521152, "cache_write_tokens": 0 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 37.8, "avg_query_seconds": 87.7, "median_query_seconds": 23.8, "avg_ttft_seconds": 9.5, "output_time_split": { "reasoning_pct": 87.7, "content_pct": 0.5, "tool_call_pct": 11.8 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 66.252, "latency": 4495.202, "stderr": 2.685, "cost_per_test": 0.713033, "token_totals": { "input_tokens": 3932720612, "output_tokens": 29249606, "reasoning_tokens": null, "cache_read_tokens": 3849386240, "cache_write_tokens": 0 }, "token_tasks_measured": 100, "turn_metrics": { "tasks_measured": 100, "avg_turns": 129.4, "avg_query_seconds": 34.7, "median_query_seconds": 10.9, "avg_ttft_seconds": 2.4, "output_time_split": { "reasoning_pct": 74.2, "content_pct": 0.4, "tool_call_pct": 25.4 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 65.477, "latency": 1195.527, "stderr": 3.691, "cost_per_test": 1.364559, "token_totals": { "input_tokens": 1482730908, "output_tokens": 25203657, "reasoning_tokens": 19717258, "cache_read_tokens": 1444841999, "cache_write_tokens": 37877007 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 56.9, "avg_query_seconds": 21.0, "median_query_seconds": 6.1, "avg_ttft_seconds": 14.8, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 65.128, "latency": 2247.236, "stderr": 2.35, "cost_per_test": 1.08141, "token_totals": { "input_tokens": 864870937, "output_tokens": 15254147, "reasoning_tokens": 11469965, "cache_read_tokens": 855589504, "cache_write_tokens": 0 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 53.6, "avg_query_seconds": 41.9, "median_query_seconds": 13.2, "avg_ttft_seconds": 3.2, "output_time_split": { "reasoning_pct": 82.9, "content_pct": 0.8, "tool_call_pct": 16.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "reflection/Beam-1.0": { "accuracy": 64.733, "latency": 1204.28, "stderr": 2.259, "cost_per_test": null, "token_totals": { "input_tokens": 905656608, "output_tokens": 6481191, "reasoning_tokens": 4044707, "cache_read_tokens": 884618112, "cache_write_tokens": null }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 83.4, "avg_query_seconds": 14.4, "median_query_seconds": 1.6, "avg_ttft_seconds": 0.5, "output_time_split": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "meta/muse-spark-open": { "accuracy": 61.983, "latency": 5078.614, "stderr": 2.704, "cost_per_test": 5.348821, "token_totals": { "input_tokens": 973042320, "output_tokens": 8495845, "reasoning_tokens": 4888586, "cache_read_tokens": 642573184, "cache_write_tokens": 0 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 56.7, "avg_query_seconds": 89.5, "median_query_seconds": 21.8, "avg_ttft_seconds": 28.0, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "inception/mercury-3": { "accuracy": 57.88, "latency": 531.279, "stderr": 2.914, "cost_per_test": 0.194746, "token_totals": { "input_tokens": 875577703, "output_tokens": 11690649, "reasoning_tokens": 11334293, "cache_read_tokens": 797615720, "cache_write_tokens": 0 }, "token_tasks_measured": 102, "turn_metrics": { "tasks_measured": 102, "avg_turns": 71.9, "avg_query_seconds": 7.4, "median_query_seconds": 3.6, "avg_ttft_seconds": 4.3, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "thinkingmachines/inkling": { "accuracy": 40.971, "latency": 1581.835, "stderr": 2.982, "cost_per_test": 1.671783, "token_totals": { "input_tokens": 658436029, "output_tokens": 9281665, "reasoning_tokens": null, "cache_read_tokens": 641195264, "cache_write_tokens": 0 }, "token_tasks_measured": 98, "turn_metrics": { "tasks_measured": 98, "avg_turns": 66.2, "avg_query_seconds": 23.9, "median_query_seconds": 5.7, "avg_ttft_seconds": 1.9, "output_time_split": { "reasoning_pct": 99.3, "content_pct": 0.6, "tool_call_pct": 0.1 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 40.823, "latency": 1173.625, "stderr": 3.21, "cost_per_test": 0.249301, "token_totals": { "input_tokens": 254752442, "output_tokens": 6362193, "reasoning_tokens": null, "cache_read_tokens": 245337472, "cache_write_tokens": 0 }, "token_tasks_measured": 101, "turn_metrics": { "tasks_measured": 101, "avg_turns": 48.0, "avg_query_seconds": 24.4, "median_query_seconds": 7.2, "avg_ttft_seconds": 2.2, "output_time_split": { "reasoning_pct": 99.9, "content_pct": 0.1, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "code_migration": { "anthropic/claude-sonnet-5-5": { "accuracy": 87.372, "latency": 14850.395, "stderr": 2.388, "cost_per_test": 45.405088, "token_totals": { "input_tokens": 20450605917, "output_tokens": 74788366, "reasoning_tokens": 34898228, "cache_read_tokens": 20245723628, "cache_write_tokens": 204790865 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 757.4, "avg_query_seconds": 13.2, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 80.487, "latency": 12635.272, "stderr": 4.08, "cost_per_test": 72.30083, "token_totals": { "input_tokens": 16604648219, "output_tokens": 56658262, "reasoning_tokens": 20731364, "cache_read_tokens": 16430817403, "cache_write_tokens": 173755296 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 625.4, "avg_query_seconds": 14.7, "median_query_seconds": 6.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 78.031, "latency": 5286.504, "stderr": 3.128, "cost_per_test": 66.641779, "token_totals": { "input_tokens": 1828192015, "output_tokens": 12132196, "reasoning_tokens": 4524551, "cache_read_tokens": 1791878139, "cache_write_tokens": 36294376 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 108.4, "avg_query_seconds": 36.4, "median_query_seconds": 19.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 75.136, "latency": 6157.942, "stderr": 3.609, "cost_per_test": 9.938473, "token_totals": { "input_tokens": 2140765806, "output_tokens": 14016797, "reasoning_tokens": 5851455, "cache_read_tokens": 2110633678, "cache_write_tokens": 30109934 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 123.4, "avg_query_seconds": 38.4, "median_query_seconds": 20.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 74.429, "latency": 8400.232, "stderr": 4.512, "cost_per_test": 44.644196, "token_totals": { "input_tokens": 8076346935, "output_tokens": 26849639, "reasoning_tokens": 5541747, "cache_read_tokens": 7868600042, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 329.3, "avg_query_seconds": 23.1, "median_query_seconds": 11.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 69.781, "latency": 9687.179, "stderr": 4.035, "cost_per_test": 47.552905, "token_totals": { "input_tokens": 11958750391, "output_tokens": 23001800, "reasoning_tokens": 10801141, "cache_read_tokens": 11889714692, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 489.6, "avg_query_seconds": 10.1, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 69.685, "latency": 4516.513, "stderr": 3.891, "cost_per_test": 6.021213, "token_totals": { "input_tokens": 3212892026, "output_tokens": 10598879, "reasoning_tokens": 3692961, "cache_read_tokens": 3186778556, "cache_write_tokens": 26081847 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 175.7, "avg_query_seconds": 14.7, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 68.5, "latency": 8133.069, "stderr": 3.476, "cost_per_test": 1.572628, "token_totals": { "input_tokens": 11188739656, "output_tokens": 22875100, "reasoning_tokens": 10958121, "cache_read_tokens": 11164665216, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 437.4, "avg_query_seconds": 5.8, "median_query_seconds": 4.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder-full": { "accuracy": 61.652, "latency": 15566.738, "stderr": 4.371, "cost_per_test": 23.236252, "token_totals": { "input_tokens": 4997748393, "output_tokens": 15267636, "reasoning_tokens": 5439846, "cache_read_tokens": 4951032316, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 255.6, "avg_query_seconds": 53.4, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 59.682, "latency": 7030.306, "stderr": 4.599, "cost_per_test": 15.708479, "token_totals": { "input_tokens": 4230470567, "output_tokens": 13267585, "reasoning_tokens": 5018950, "cache_read_tokens": 4179632896, "cache_write_tokens": 49424116 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 187.7, "avg_query_seconds": 26.3, "median_query_seconds": 17.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 54.532, "latency": 14749.791, "stderr": 3.634, "cost_per_test": 25.858914, "token_totals": { "input_tokens": 5817056303, "output_tokens": 21718961, "reasoning_tokens": 10007193, "cache_read_tokens": 5774281600, "cache_write_tokens": null }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 288.1, "avg_query_seconds": 40.9, "median_query_seconds": 24.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 54.075, "latency": 10660.633, "stderr": 3.209, "cost_per_test": 33.523556, "token_totals": { "input_tokens": 7674828776, "output_tokens": 21022679, "reasoning_tokens": 11007918, "cache_read_tokens": 7637999632, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 379.8, "avg_query_seconds": 16.5, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 52.028, "latency": 10883.534, "stderr": 3.359, "cost_per_test": 11.806673, "token_totals": { "input_tokens": 14146599509, "output_tokens": 27289057, "reasoning_tokens": 11810197, "cache_read_tokens": 14118838528, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 571.4, "avg_query_seconds": 10.8, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.7": { "accuracy": 51.821, "latency": 3787.561, "stderr": 3.201, "cost_per_test": 32.981557, "token_totals": { "input_tokens": 2223002903, "output_tokens": 10695629, "reasoning_tokens": 4828398, "cache_read_tokens": 2192254080, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 168.0, "avg_query_seconds": 14.5, "median_query_seconds": 8.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 50.625, "latency": 12036.086, "stderr": 3.999, "cost_per_test": 4.135786, "token_totals": { "input_tokens": 15826013180, "output_tokens": 26129234, "reasoning_tokens": 11647712, "cache_read_tokens": 15383262776, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 672.2, "avg_query_seconds": 8.3, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 50.199, "latency": 5148.212, "stderr": 4.208, "cost_per_test": 6.130933, "token_totals": { "input_tokens": 5608709357, "output_tokens": 30123861, "reasoning_tokens": 17474500, "cache_read_tokens": 5550232078, "cache_write_tokens": 58450205 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 224.4, "avg_query_seconds": 11.0, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 48.195, "latency": 7363.872, "stderr": 3.35, "cost_per_test": 1.729595, "token_totals": { "input_tokens": 6238818064, "output_tokens": 20288643, "reasoning_tokens": 12068935, "cache_read_tokens": 6202642760, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 328.0, "avg_query_seconds": 10.0, "median_query_seconds": 6.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 47.073, "latency": 12258.712, "stderr": 4.707, "cost_per_test": 0.891745, "token_totals": { "input_tokens": 4617068362, "output_tokens": 21009209, "reasoning_tokens": 12444641, "cache_read_tokens": 4569710080, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 250.5, "avg_query_seconds": 40.6, "median_query_seconds": 21.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 45.451, "latency": 3543.575, "stderr": 4.218, "cost_per_test": 0.708851, "token_totals": { "input_tokens": 2188525650, "output_tokens": 12053918, "reasoning_tokens": 7187113, "cache_read_tokens": 2166536585, "cache_write_tokens": 21964120 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 138.6, "avg_query_seconds": 18.9, "median_query_seconds": 11.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 44.695, "latency": 6574.27, "stderr": 4.376, "cost_per_test": 3.421469, "token_totals": { "input_tokens": 3364146831, "output_tokens": 13818897, "reasoning_tokens": 6708614, "cache_read_tokens": 3310263989, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 216.5, "avg_query_seconds": 19.9, "median_query_seconds": 13.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Openrouter", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 38.764, "latency": 14739.113, "stderr": 4.306, "cost_per_test": 0.612992, "token_totals": { "input_tokens": 7977445980, "output_tokens": 31336095, "reasoning_tokens": 18580673, "cache_read_tokens": 7915208384, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 326.9, "avg_query_seconds": 36.8, "median_query_seconds": 20.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 38.173, "latency": 7443.728, "stderr": 4.282, "cost_per_test": 8.31345, "token_totals": { "input_tokens": 5623048121, "output_tokens": 12806610, "reasoning_tokens": 6251130, "cache_read_tokens": 5518738671, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 358.0, "avg_query_seconds": 17.0, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-3": { "accuracy": 36.882, "latency": 5686.628, "stderr": 3.641, "cost_per_test": 2.206179, "token_totals": { "input_tokens": 5861640240, "output_tokens": 16809801, "reasoning_tokens": 15199410, "cache_read_tokens": 4970682242, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 352.6, "avg_query_seconds": 9.9, "median_query_seconds": 4.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "meta/muse-spark-open": { "accuracy": 25.053, "latency": 6556.205, "stderr": 3.48, "cost_per_test": 4.642276, "token_totals": { "input_tokens": 742293401, "output_tokens": 5299693, "reasoning_tokens": 2643561, "cache_read_tokens": 594060912, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 101.0, "avg_query_seconds": 59.9, "median_query_seconds": 17.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 22.669, "latency": 16267.502, "stderr": 3.551, "cost_per_test": 19.510208, "token_totals": { "input_tokens": 3064615804, "output_tokens": 14435317, "reasoning_tokens": 6152061, "cache_read_tokens": 2993379072, "cache_write_tokens": 69570492 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 225.2, "avg_query_seconds": 68.8, "median_query_seconds": 53.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 22.566, "latency": 713.542, "stderr": 3.465, "cost_per_test": null, "token_totals": { "input_tokens": 571195376, "output_tokens": 2794524, "reasoning_tokens": 1413386, "cache_read_tokens": 562517824, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 130.0, "avg_query_seconds": 1.3, "median_query_seconds": 0.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 10.398, "latency": 1133.764, "stderr": 2.373, "cost_per_test": 1.587286, "token_totals": { "input_tokens": 1538192882, "output_tokens": 5866691, "reasoning_tokens": 0, "cache_read_tokens": 1524835840, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 273.5, "avg_query_seconds": 2.8, "median_query_seconds": 2.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 8.57, "latency": 1413.991, "stderr": 2.746, "cost_per_test": 6.017274, "token_totals": { "input_tokens": 1501371349, "output_tokens": 5071588, "reasoning_tokens": 0, "cache_read_tokens": 1487993088, "cache_write_tokens": 0 }, "token_tasks_measured": 60, "turn_metrics": { "tasks_measured": 60, "avg_turns": 278.5, "avg_query_seconds": 2.7, "median_query_seconds": 1.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "legal_research": { "google/gemini-4-argon": { "accuracy": 54.808, "latency": 1881.045, "stderr": 3.459, "cost_per_test": 6.44908, "token_totals": { "input_tokens": 915877177, "output_tokens": 24432059, "reasoning_tokens": 11886477, "cache_read_tokens": 739668768, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 38.2, "avg_query_seconds": 49.2, "median_query_seconds": 16.4, "avg_ttft_seconds": 8.1, "output_time_split": { "reasoning_pct": 79.0, "content_pct": 3.0, "tool_call_pct": 18.1 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 53.365, "latency": 1654.378, "stderr": 3.467, "cost_per_test": 3.595401, "token_totals": { "input_tokens": 760777228, "output_tokens": 19689738, "reasoning_tokens": 13012367, "cache_read_tokens": 707647460, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 45.3, "avg_query_seconds": 36.5, "median_query_seconds": 9.9, "avg_ttft_seconds": 8.2, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 50.481, "latency": 927.111, "stderr": 3.475, "cost_per_test": 3.980242, "token_totals": { "input_tokens": 732491974, "output_tokens": 17268053, "reasoning_tokens": 11668590, "cache_read_tokens": 644062808, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 46.7, "avg_query_seconds": 19.9, "median_query_seconds": 7.2, "avg_ttft_seconds": 5.5, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 50.481, "latency": 6749.531, "stderr": 3.475, "cost_per_test": 25.163725, "token_totals": { "input_tokens": 1686854010, "output_tokens": 141554772, "reasoning_tokens": 123959393, "cache_read_tokens": 1436953409, "cache_write_tokens": 249809865 }, "token_tasks_measured": 185, "turn_metrics": { "tasks_measured": 197, "avg_turns": 72.9, "avg_query_seconds": 89.0, "median_query_seconds": 9.2, "avg_ttft_seconds": 9.7, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 49.038, "latency": 3110.171, "stderr": 3.475, "cost_per_test": 2.241388, "token_totals": { "input_tokens": 458279584, "output_tokens": 55494507, "reasoning_tokens": null, "cache_read_tokens": 368033787, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 23.3, "avg_query_seconds": 133.6, "median_query_seconds": 67.1, "avg_ttft_seconds": 3.4, "output_time_split": { "reasoning_pct": 82.2, "content_pct": 1.1, "tool_call_pct": 16.7 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 48.077, "latency": 5588.989, "stderr": 3.473, "cost_per_test": 14.826571, "token_totals": { "input_tokens": 2650038387, "output_tokens": 173348390, "reasoning_tokens": 151205430, "cache_read_tokens": 2338418360, "cache_write_tokens": 311466299 }, "token_tasks_measured": 201, "turn_metrics": { "tasks_measured": 204, "avg_turns": 85.2, "avg_query_seconds": 64.6, "median_query_seconds": 11.2, "avg_ttft_seconds": 8.2, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 47.115, "latency": 1274.337, "stderr": 3.469, "cost_per_test": 4.920812, "token_totals": { "input_tokens": 476673717, "output_tokens": 14028919, "reasoning_tokens": 9121242, "cache_read_tokens": 350504320, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 44.0, "avg_query_seconds": 28.9, "median_query_seconds": 12.5, "avg_ttft_seconds": 3.3, "output_time_split": { "reasoning_pct": 78.7, "content_pct": 12.9, "tool_call_pct": 8.5 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 47.115, "latency": 1816.322, "stderr": 3.469, "cost_per_test": 0.181201, "token_totals": { "input_tokens": 246448526, "output_tokens": 20667764, "reasoning_tokens": 15354087, "cache_read_tokens": 204079360, "cache_write_tokens": 0 }, "token_tasks_measured": 205, "turn_metrics": { "tasks_measured": 206, "avg_turns": 21.3, "avg_query_seconds": 85.4, "median_query_seconds": 43.3, "avg_ttft_seconds": 5.2, "output_time_split": { "reasoning_pct": 81.4, "content_pct": 1.9, "tool_call_pct": 16.6 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 46.154, "latency": 760.122, "stderr": 3.465, "cost_per_test": 3.469601, "token_totals": { "input_tokens": 672994446, "output_tokens": 6248749, "reasoning_tokens": 3873135, "cache_read_tokens": 651592955, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 37.2, "avg_query_seconds": 20.4, "median_query_seconds": 10.2, "avg_ttft_seconds": 4.2, "output_time_split": { "reasoning_pct": 61.9, "content_pct": 2.6, "tool_call_pct": 35.5 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 46.154, "latency": 7649.584, "stderr": 3.465, "cost_per_test": 5.312295, "token_totals": { "input_tokens": 1691431541, "output_tokens": 63439847, "reasoning_tokens": 53571199, "cache_read_tokens": 1582915968, "cache_write_tokens": null }, "token_tasks_measured": 187, "turn_metrics": { "tasks_measured": 187, "avg_turns": 72.2, "avg_query_seconds": 105.9, "median_query_seconds": 44.3, "avg_ttft_seconds": 21.5, "output_time_split": { "reasoning_pct": 81.4, "content_pct": 0.2, "tool_call_pct": 18.5 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 45.192, "latency": 2106.881, "stderr": 3.459, "cost_per_test": 0.69856, "token_totals": { "input_tokens": 2003160078, "output_tokens": 87441264, "reasoning_tokens": 76652436, "cache_read_tokens": 1909100928, "cache_write_tokens": 0 }, "token_tasks_measured": 207, "turn_metrics": { "tasks_measured": 207, "avg_turns": 74.5, "avg_query_seconds": 28.2, "median_query_seconds": 6.4, "avg_ttft_seconds": 1.6, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 45.192, "latency": 3067.486, "stderr": 3.459, "cost_per_test": 0.120196, "token_totals": { "input_tokens": 636228822, "output_tokens": 43453265, "reasoning_tokens": null, "cache_read_tokens": 559661440, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 30.9, "avg_query_seconds": 99.2, "median_query_seconds": 40.1, "avg_ttft_seconds": 1.8, "output_time_split": { "reasoning_pct": 85.5, "content_pct": 0.9, "tool_call_pct": 13.6 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 45.192, "latency": 3839.511, "stderr": 3.459, "cost_per_test": 0.846372, "token_totals": { "input_tokens": 312749338, "output_tokens": 50001075, "reasoning_tokens": 43836379, "cache_read_tokens": 264949824, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 31.0, "avg_query_seconds": 123.9, "median_query_seconds": 36.1, "avg_ttft_seconds": 3.4, "output_time_split": { "reasoning_pct": 72.0, "content_pct": 8.7, "tool_call_pct": 19.3 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 43.75, "latency": 4979.806, "stderr": 3.448, "cost_per_test": 1.647359, "token_totals": { "input_tokens": 1674824268, "output_tokens": 253184033, "reasoning_tokens": 237163492, "cache_read_tokens": 1419547906, "cache_write_tokens": 255154225 }, "token_tasks_measured": 206, "turn_metrics": { "tasks_measured": 206, "avg_turns": 60.8, "avg_query_seconds": 81.9, "median_query_seconds": 14.1, "avg_ttft_seconds": 8.8, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 40.865, "latency": 2527.177, "stderr": 3.417, "cost_per_test": 1.088017, "token_totals": { "input_tokens": 433434003, "output_tokens": 35408474, "reasoning_tokens": 30213976, "cache_read_tokens": 380911360, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 30.2, "avg_query_seconds": 83.7, "median_query_seconds": 27.7, "avg_ttft_seconds": 2.4, "output_time_split": { "reasoning_pct": 76.2, "content_pct": 1.7, "tool_call_pct": 22.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 39.423, "latency": 1491.339, "stderr": 3.397, "cost_per_test": 10.577143, "token_totals": { "input_tokens": 290661092, "output_tokens": 21448775, "reasoning_tokens": 16238030, "cache_read_tokens": 215071044, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 19.0, "avg_query_seconds": 78.6, "median_query_seconds": 31.7, "avg_ttft_seconds": 9.4, "output_time_split": { "reasoning_pct": 25.5, "content_pct": 0.6, "tool_call_pct": 74.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 38.942, "latency": 324.318, "stderr": 3.389, "cost_per_test": 1.630498, "token_totals": { "input_tokens": 581263116, "output_tokens": 14420656, "reasoning_tokens": 10782340, "cache_read_tokens": 474745174, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 50.6, "avg_query_seconds": 6.4, "median_query_seconds": 1.8, "avg_ttft_seconds": 1.3, "output_time_split": { "reasoning_pct": 98.4, "content_pct": 0.0, "tool_call_pct": 1.6 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 38.462, "latency": 1763.023, "stderr": 3.381, "cost_per_test": 1.559385, "token_totals": { "input_tokens": 1351006836, "output_tokens": 6986771, "reasoning_tokens": 3248800, "cache_read_tokens": 1267363808, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 31.6, "avg_query_seconds": 55.8, "median_query_seconds": 10.0, "avg_ttft_seconds": 11.0, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 38.462, "latency": 2060.48, "stderr": 3.381, "cost_per_test": 3.384561, "token_totals": { "input_tokens": 329856713, "output_tokens": 20763888, "reasoning_tokens": 15434407, "cache_read_tokens": 260063643, "cache_write_tokens": 68892844 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 21.7, "avg_query_seconds": 95.1, "median_query_seconds": 43.9, "avg_ttft_seconds": 13.1, "output_time_split": { "reasoning_pct": 24.3, "content_pct": 0.9, "tool_call_pct": 74.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 37.981, "latency": 919.689, "stderr": 3.373, "cost_per_test": 0.070424, "token_totals": { "input_tokens": 365192541, "output_tokens": 17884630, "reasoning_tokens": 10104423, "cache_read_tokens": 305459072, "cache_write_tokens": 0 }, "token_tasks_measured": 202, "turn_metrics": { "tasks_measured": 205, "avg_turns": 27.4, "avg_query_seconds": 35.9, "median_query_seconds": 19.2, "avg_ttft_seconds": 3.5, "output_time_split": { "reasoning_pct": 0.7, "content_pct": 0.0, "tool_call_pct": 99.3 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 37.5, "latency": 6237.394, "stderr": 3.365, "cost_per_test": 7.231463, "token_totals": { "input_tokens": 1110222674, "output_tokens": 53232219, "reasoning_tokens": 44000137, "cache_read_tokens": 842326272, "cache_write_tokens": 266448109 }, "token_tasks_measured": 171, "turn_metrics": { "tasks_measured": 171, "avg_turns": 44.1, "avg_query_seconds": 141.6, "median_query_seconds": 25.7, "avg_ttft_seconds": 7.8, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 34.135, "latency": 2911.983, "stderr": 3.296, "cost_per_test": 0.444777, "token_totals": { "input_tokens": 1230685744, "output_tokens": 102073269, "reasoning_tokens": 90583933, "cache_read_tokens": 1135801046, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 54.4, "avg_query_seconds": 53.5, "median_query_seconds": 16.7, "avg_ttft_seconds": 1.3, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-luna": { "accuracy": 30.288, "latency": 2547.747, "stderr": 3.194, "cost_per_test": 0.440296, "token_totals": { "input_tokens": 1143721770, "output_tokens": 59694127, "reasoning_tokens": 54603753, "cache_read_tokens": 1042899991, "cache_write_tokens": 99842719 }, "token_tasks_measured": 206, "turn_metrics": { "tasks_measured": 208, "avg_turns": 38.0, "avg_query_seconds": 67.1, "median_query_seconds": 16.9, "avg_ttft_seconds": 7.2, "output_time_split": { "reasoning_pct": 66.4, "content_pct": 0.0, "tool_call_pct": 33.6 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 28.846, "latency": 94.568, "stderr": 3.149, "cost_per_test": null, "token_totals": { "input_tokens": 394030584, "output_tokens": 5402310, "reasoning_tokens": 3403372, "cache_read_tokens": 359513792, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 36.4, "avg_query_seconds": 2.6, "median_query_seconds": 1.3, "avg_ttft_seconds": 0.4, "output_time_split": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "inception/mercury-3": { "accuracy": 28.846, "latency": 446.517, "stderr": 3.149, "cost_per_test": 0.074351, "token_totals": { "input_tokens": 271885585, "output_tokens": 22851545, "reasoning_tokens": 20198714, "cache_read_tokens": 219955154, "cache_write_tokens": 0 }, "token_tasks_measured": 207, "turn_metrics": { "tasks_measured": 208, "avg_turns": 26.3, "avg_query_seconds": 16.9, "median_query_seconds": 6.1, "avg_ttft_seconds": 3.3, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "thinkingmachines/inkling": { "accuracy": 28.365, "latency": 869.377, "stderr": 3.133, "cost_per_test": 0.494044, "token_totals": { "input_tokens": 271552348, "output_tokens": 8856526, "reasoning_tokens": null, "cache_read_tokens": 246578560, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 37.8, "avg_query_seconds": 23.0, "median_query_seconds": 4.5, "avg_ttft_seconds": 2.7, "output_time_split": { "reasoning_pct": 99.9, "content_pct": 0.0, "tool_call_pct": 0.1 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 25.962, "latency": 4202.728, "stderr": 3.047, "cost_per_test": 3.220614, "token_totals": { "input_tokens": 1392458117, "output_tokens": 69708508, "reasoning_tokens": 62148334, "cache_read_tokens": 1260829967, "cache_write_tokens": 130975483 }, "token_tasks_measured": 202, "turn_metrics": { "tasks_measured": 202, "avg_turns": 35.9, "avg_query_seconds": 117.1, "median_query_seconds": 21.4, "avg_ttft_seconds": 8.0, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 25.481, "latency": 774.823, "stderr": 3.029, "cost_per_test": 0.141548, "token_totals": { "input_tokens": 209190661, "output_tokens": 10262319, "reasoning_tokens": null, "cache_read_tokens": 190714496, "cache_write_tokens": 0 }, "token_tasks_measured": 207, "turn_metrics": { "tasks_measured": 207, "avg_turns": 34.6, "avg_query_seconds": 22.4, "median_query_seconds": 5.1, "avg_ttft_seconds": 1.9, "output_time_split": { "reasoning_pct": 100.0, "content_pct": 0.0, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "legal_agent_benchmark": { "google/gemini-4-argon": { "accuracy": 27.083, "latency": 1755.397, "stderr": 3.738, "cost_per_test": 10.54836, "token_totals": { "input_tokens": 812145091, "output_tokens": 18424207, "reasoning_tokens": 5211457, "cache_read_tokens": 733823568, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 23.9, "avg_query_seconds": 70.3, "median_query_seconds": 12.8, "avg_ttft_seconds": 13.1, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 25.0, "latency": 14355.684, "stderr": 3.599, "cost_per_test": 13.6316, "token_totals": { "input_tokens": 2657733432, "output_tokens": 13857658, "reasoning_tokens": 6185749, "cache_read_tokens": 1586608896, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 80.6, "avg_query_seconds": 107.7, "median_query_seconds": 20.4, "avg_ttft_seconds": 65.0, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse-wonder": { "accuracy": 23.75, "latency": 906.14, "stderr": 3.646, "cost_per_test": 7.081987, "token_totals": { "input_tokens": 1414833931, "output_tokens": 12111113, "reasoning_tokens": 1875186, "cache_read_tokens": 1374979944, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 50.0, "avg_query_seconds": 16.8, "median_query_seconds": 5.3, "avg_ttft_seconds": 10.2, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 23.333, "latency": 2649.375, "stderr": 3.547, "cost_per_test": 5.413598, "token_totals": { "input_tokens": 1349672470, "output_tokens": 13499459, "reasoning_tokens": 2307478, "cache_read_tokens": 1308495584, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 47.6, "avg_query_seconds": 53.6, "median_query_seconds": 15.2, "avg_ttft_seconds": 25.0, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 18.333, "latency": 2148.428, "stderr": 3.183, "cost_per_test": 10.746259, "token_totals": { "input_tokens": 1088222870, "output_tokens": 20549074, "reasoning_tokens": 8836231, "cache_read_tokens": 1043005952, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 40.8, "avg_query_seconds": 51.4, "median_query_seconds": 8.6, "avg_ttft_seconds": 9.6, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 15.417, "latency": 2950.56, "stderr": 3.063, "cost_per_test": 3.358663, "token_totals": { "input_tokens": 934205246, "output_tokens": 14880894, "reasoning_tokens": 6584092, "cache_read_tokens": 888375040, "cache_write_tokens": null }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 35.7, "avg_query_seconds": 81.2, "median_query_seconds": 26.1, "avg_ttft_seconds": 11.8, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 14.583, "latency": 1911.01, "stderr": 2.862, "cost_per_test": 3.987774, "token_totals": { "input_tokens": 1376154444, "output_tokens": 16202326, "reasoning_tokens": 7676761, "cache_read_tokens": 1332783858, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 50.4, "avg_query_seconds": 36.6, "median_query_seconds": 7.8, "avg_ttft_seconds": 3.2, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 14.167, "latency": 1328.108, "stderr": 2.787, "cost_per_test": 0.612596, "token_totals": { "input_tokens": 5524481146, "output_tokens": 25881985, "reasoning_tokens": 10520362, "cache_read_tokens": 5492827392, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 146.9, "avg_query_seconds": 7.4, "median_query_seconds": 3.0, "avg_ttft_seconds": 1.8, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 13.75, "latency": 1861.848, "stderr": 2.836, "cost_per_test": 0.216614, "token_totals": { "input_tokens": 631569981, "output_tokens": 10405619, "reasoning_tokens": 4622671, "cache_read_tokens": 604899072, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 35.8, "avg_query_seconds": 51.1, "median_query_seconds": 13.1, "avg_ttft_seconds": 5.1, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 13.333, "latency": 1342.563, "stderr": 2.821, "cost_per_test": 0.085646, "token_totals": { "input_tokens": 958015494, "output_tokens": 12423312, "reasoning_tokens": 6316080, "cache_read_tokens": 925243136, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 39.9, "avg_query_seconds": 32.4, "median_query_seconds": 9.2, "avg_ttft_seconds": 4.6, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 12.917, "latency": 2556.302, "stderr": 2.807, "cost_per_test": 4.144144, "token_totals": { "input_tokens": 887032678, "output_tokens": 10027338, "reasoning_tokens": 3826649, "cache_read_tokens": 857115099, "cache_write_tokens": 29371546 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 38.0, "avg_query_seconds": 66.3, "median_query_seconds": 20.6, "avg_ttft_seconds": 9.6, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 12.083, "latency": 1161.715, "stderr": 2.579, "cost_per_test": 3.656207, "token_totals": { "input_tokens": 1288268290, "output_tokens": 16504243, "reasoning_tokens": 6011153, "cache_read_tokens": 1232055515, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 62.6, "avg_query_seconds": 17.7, "median_query_seconds": 5.4, "avg_ttft_seconds": 4.6, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 11.667, "latency": 4373.383, "stderr": 2.629, "cost_per_test": 9.004671, "token_totals": { "input_tokens": 3826336436, "output_tokens": 21461832, "reasoning_tokens": 7740132, "cache_read_tokens": 3770405951, "cache_write_tokens": 54641257 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 92.4, "avg_query_seconds": 45.8, "median_query_seconds": 17.3, "avg_ttft_seconds": 11.4, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 10.833, "latency": 655.413, "stderr": 2.723, "cost_per_test": 0.802111, "token_totals": { "input_tokens": 746916941, "output_tokens": 9384470, "reasoning_tokens": 4362757, "cache_read_tokens": 726363264, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 33.7, "avg_query_seconds": 18.3, "median_query_seconds": 4.3, "avg_ttft_seconds": 2.0, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 10.417, "latency": 744.843, "stderr": 2.572, "cost_per_test": 0.323243, "token_totals": { "input_tokens": 1006432418, "output_tokens": 9841376, "reasoning_tokens": 3565890, "cache_read_tokens": 975803052, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 48.3, "avg_query_seconds": 14.4, "median_query_seconds": 4.5, "avg_ttft_seconds": 2.7, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 10.417, "latency": 4397.953, "stderr": 2.572, "cost_per_test": 16.612583, "token_totals": { "input_tokens": 7599893900, "output_tokens": 71504693, "reasoning_tokens": 47513756, "cache_read_tokens": 7328811056, "cache_write_tokens": 271040714 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 150.0, "avg_query_seconds": 27.9, "median_query_seconds": 7.3, "avg_ttft_seconds": 25.2, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 9.167, "latency": 3507.428, "stderr": 2.366, "cost_per_test": 21.37949, "token_totals": { "input_tokens": 3749613119, "output_tokens": 44884808, "reasoning_tokens": 27684473, "cache_read_tokens": 3547556427, "cache_write_tokens": 202035592 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 75.1, "avg_query_seconds": 45.4, "median_query_seconds": 9.9, "avg_ttft_seconds": 37.2, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 8.75, "latency": 1989.335, "stderr": 2.187, "cost_per_test": 0.98378, "token_totals": { "input_tokens": 1373541600, "output_tokens": 12484050, "reasoning_tokens": 4013313, "cache_read_tokens": 1314797886, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 54.0, "avg_query_seconds": 35.5, "median_query_seconds": 11.0, "avg_ttft_seconds": 5.8, "output_time_split": null }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 7.5, "latency": 1497.601, "stderr": 2.265, "cost_per_test": 0.472848, "token_totals": { "input_tokens": 2730849267, "output_tokens": 20117381, "reasoning_tokens": 6681765, "cache_read_tokens": 2703378118, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 85.7, "avg_query_seconds": 15.9, "median_query_seconds": 4.4, "avg_ttft_seconds": 2.3, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-astra": { "accuracy": 7.083, "latency": 1953.362, "stderr": 1.899, "cost_per_test": 25.460295, "token_totals": { "input_tokens": 1310790488, "output_tokens": 10401199, "reasoning_tokens": 3452073, "cache_read_tokens": 1276652993, "cache_write_tokens": 34120712 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 45.9, "avg_query_seconds": 40.9, "median_query_seconds": 8.9, "avg_ttft_seconds": 16.5, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 6.25, "latency": 2425.625, "stderr": 1.83, "cost_per_test": 3.536302, "token_totals": { "input_tokens": 1227717408, "output_tokens": 11158751, "reasoning_tokens": 3901277, "cache_read_tokens": 1191414163, "cache_write_tokens": 36281197 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 43.3, "avg_query_seconds": 53.8, "median_query_seconds": 10.2, "avg_ttft_seconds": 22.0, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-3": { "accuracy": 5.0, "latency": 886.884, "stderr": 1.61, "cost_per_test": 0.188085, "token_totals": { "input_tokens": 659070144, "output_tokens": 6092886, "reasoning_tokens": 5836559, "cache_read_tokens": 505214083, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 33.6, "avg_query_seconds": 7.9, "median_query_seconds": 3.2, "avg_ttft_seconds": 5.2, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "reflection/Beam-1.0": { "accuracy": 4.583, "latency": 116.883, "stderr": 1.775, "cost_per_test": null, "token_totals": { "input_tokens": 540211238, "output_tokens": 4325040, "reasoning_tokens": 1196691, "cache_read_tokens": 509200704, "cache_write_tokens": null }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 34.0, "avg_query_seconds": 2.6, "median_query_seconds": 0.8, "avg_ttft_seconds": 0.5, "output_time_split": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "openai/gpt-6-luna": { "accuracy": 4.167, "latency": 1305.488, "stderr": 1.629, "cost_per_test": 0.298018, "token_totals": { "input_tokens": 1322468906, "output_tokens": 12364131, "reasoning_tokens": 6724988, "cache_read_tokens": 1277646907, "cache_write_tokens": 44806375 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 41.2, "avg_query_seconds": 30.2, "median_query_seconds": 6.0, "avg_ttft_seconds": 5.7, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 4.167, "latency": 2620.795, "stderr": 1.518, "cost_per_test": 4.264376, "token_totals": { "input_tokens": 4904885383, "output_tokens": 69221549, "reasoning_tokens": 47929104, "cache_read_tokens": 4716242036, "cache_write_tokens": 188617435 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 99.4, "avg_query_seconds": 25.2, "median_query_seconds": 7.1, "avg_ttft_seconds": 19.2, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 3.75, "latency": 529.644, "stderr": 1.469, "cost_per_test": 1.252023, "token_totals": { "input_tokens": 538710546, "output_tokens": 7813620, "reasoning_tokens": null, "cache_read_tokens": 513145856, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 35.4, "avg_query_seconds": 13.8, "median_query_seconds": 1.7, "avg_ttft_seconds": 1.9, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 2.917, "latency": 455.265, "stderr": 1.362, "cost_per_test": 0.17869, "token_totals": { "input_tokens": 200497219, "output_tokens": 5802466, "reasoning_tokens": null, "cache_read_tokens": 190288768, "cache_write_tokens": 0 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 32.4, "avg_query_seconds": 13.6, "median_query_seconds": 2.1, "avg_ttft_seconds": 1.5, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 1.667, "latency": 821.505, "stderr": 1.013, "cost_per_test": 1.477673, "token_totals": { "input_tokens": 729881787, "output_tokens": 7345258, "reasoning_tokens": 3332139, "cache_read_tokens": 676528604, "cache_write_tokens": 53342683 }, "token_tasks_measured": 120, "turn_metrics": { "tasks_measured": 120, "avg_turns": 29.0, "avg_query_seconds": 26.4, "median_query_seconds": 8.3, "avg_ttft_seconds": 7.8, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null } }, "terminal_bench_4": { "anthropic/claude-opus-5-5": { "accuracy": 65.152, "latency": 3810.852, "stderr": 0.0, "cost_per_test": 13.198632, "token_totals": { "input_tokens": 1850938631, "output_tokens": 14913437, "reasoning_tokens": 10690470, "cache_read_tokens": 1824613192, "cache_write_tokens": 26310456 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 112.2, "avg_query_seconds": 22.3, "median_query_seconds": 10.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 64.141, "latency": 4928.927, "stderr": 1.01, "cost_per_test": 16.513006, "token_totals": { "input_tokens": 3839535071, "output_tokens": 23509474, "reasoning_tokens": 17732859, "cache_read_tokens": 3801766853, "cache_write_tokens": 37747686 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 154.2, "avg_query_seconds": 19.2, "median_query_seconds": 9.4, "avg_ttft_seconds": 18.6, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 63.636, "latency": 2960.659, "stderr": 1.75, "cost_per_test": 4.56894, "token_totals": { "input_tokens": 1087338892, "output_tokens": 6110085, "reasoning_tokens": 3817315, "cache_read_tokens": 1074571435, "cache_write_tokens": 12131879 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 77.5, "avg_query_seconds": 29.7, "median_query_seconds": 18.0, "avg_ttft_seconds": 8.4, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 59.596, "latency": 2142.082, "stderr": 4.403, "cost_per_test": 9.584259, "token_totals": { "input_tokens": 193308076, "output_tokens": 5366501, "reasoning_tokens": 3845988, "cache_read_tokens": 184898940, "cache_write_tokens": 8367663 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 29.8, "avg_query_seconds": 62.8, "median_query_seconds": 30.3, "avg_ttft_seconds": 45.1, "output_time_split": { "reasoning_pct": 5.7, "content_pct": 15.8, "tool_call_pct": 78.4 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 58.081, "latency": 4330.665, "stderr": 3.312, "cost_per_test": 10.528007, "token_totals": { "input_tokens": 2090144775, "output_tokens": 9938507, "reasoning_tokens": 6960300, "cache_read_tokens": 2069605468, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 139.8, "avg_query_seconds": 23.5, "median_query_seconds": 10.8, "avg_ttft_seconds": 21.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 57.576, "latency": 3214.793, "stderr": 2.314, "cost_per_test": 17.63994, "token_totals": { "input_tokens": 1088824555, "output_tokens": 10645668, "reasoning_tokens": 6605548, "cache_read_tokens": 895783049, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 73.5, "avg_query_seconds": 35.6, "median_query_seconds": 20.8, "avg_ttft_seconds": 12.6, "output_time_split": { "reasoning_pct": 65.7, "content_pct": 33.9, "tool_call_pct": 0.4 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 55.051, "latency": 2711.667, "stderr": 1.821, "cost_per_test": 1.72496, "token_totals": { "input_tokens": 182186678, "output_tokens": 6309120, "reasoning_tokens": 4640611, "cache_read_tokens": 172615332, "cache_write_tokens": 9543063 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 28.9, "avg_query_seconds": 81.9, "median_query_seconds": 37.2, "avg_ttft_seconds": 58.8, "output_time_split": { "reasoning_pct": 5.3, "content_pct": 14.8, "tool_call_pct": 79.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 49.495, "latency": 1965.137, "stderr": 1.821, "cost_per_test": 2.039693, "token_totals": { "input_tokens": 1021791609, "output_tokens": 7098058, "reasoning_tokens": 4577914, "cache_read_tokens": 1009082857, "cache_write_tokens": 12671324 }, "token_tasks_measured": 66, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 49.495, "latency": 3280.687, "stderr": 2.201, "cost_per_test": 14.657694, "token_totals": { "input_tokens": 2276758314, "output_tokens": 10781411, "reasoning_tokens": 7812084, "cache_read_tokens": 2198750960, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 146.6, "avg_query_seconds": 15.9, "median_query_seconds": 7.6, "avg_ttft_seconds": 14.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 38.889, "latency": 4952.947, "stderr": 1.821, "cost_per_test": 9.37436, "token_totals": { "input_tokens": 2068794376, "output_tokens": 12825626, "reasoning_tokens": 9581216, "cache_read_tokens": 2047400960, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 146.1, "avg_query_seconds": 20.0, "median_query_seconds": 9.3, "avg_ttft_seconds": 5.2, "output_time_split": { "reasoning_pct": 98.2, "content_pct": 1.8, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 38.889, "latency": 8591.435, "stderr": 1.336, "cost_per_test": 12.146852, "token_totals": { "input_tokens": 2566775634, "output_tokens": 17868625, "reasoning_tokens": 13675033, "cache_read_tokens": 2536611883, "cache_write_tokens": null }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 141.0, "avg_query_seconds": 51.5, "median_query_seconds": 32.5, "avg_ttft_seconds": 27.5, "output_time_split": { "reasoning_pct": 79.8, "content_pct": 0.7, "tool_call_pct": 19.5 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 37.374, "latency": 3681.073, "stderr": 2.672, "cost_per_test": 1.219907, "token_totals": { "input_tokens": 2341231330, "output_tokens": 12191289, "reasoning_tokens": 8946023, "cache_read_tokens": 2306387619, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 162.2, "avg_query_seconds": 12.6, "median_query_seconds": 5.3, "avg_ttft_seconds": 2.3, "output_time_split": { "reasoning_pct": 78.3, "content_pct": 0.6, "tool_call_pct": 21.1 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 36.364, "latency": 3116.385, "stderr": 2.314, "cost_per_test": 3.117464, "token_totals": { "input_tokens": 2981637313, "output_tokens": 21606148, "reasoning_tokens": 16498863, "cache_read_tokens": 2952152829, "cache_write_tokens": 29466242 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 137.1, "avg_query_seconds": 11.3, "median_query_seconds": 4.7, "avg_ttft_seconds": 9.5, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 31.818, "latency": 2470.449, "stderr": 2.624, "cost_per_test": 0.539359, "token_totals": { "input_tokens": 2602850452, "output_tokens": 14064073, "reasoning_tokens": 10223328, "cache_read_tokens": 2592293631, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 162.5, "avg_query_seconds": 7.7, "median_query_seconds": 4.4, "avg_ttft_seconds": 1.7, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 31.313, "latency": 9720.048, "stderr": 3.072, "cost_per_test": 0.456609, "token_totals": { "input_tokens": 1740494990, "output_tokens": 15117928, "reasoning_tokens": 12015515, "cache_read_tokens": 1715650773, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 108.9, "avg_query_seconds": 64.8, "median_query_seconds": 31.2, "avg_ttft_seconds": 16.7, "output_time_split": { "reasoning_pct": 87.3, "content_pct": 1.7, "tool_call_pct": 11.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 28.788, "latency": 3007.523, "stderr": 2.314, "cost_per_test": 18.011582, "token_totals": { "input_tokens": 1192820725, "output_tokens": 10497000, "reasoning_tokens": 6887226, "cache_read_tokens": 1145959595, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 95.2, "avg_query_seconds": 24.4, "median_query_seconds": 12.9, "avg_ttft_seconds": 4.1, "output_time_split": { "reasoning_pct": 94.7, "content_pct": 5.3, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 25.253, "latency": 3030.551, "stderr": 1.01, "cost_per_test": 0.734722, "token_totals": { "input_tokens": 2497428556, "output_tokens": 16603505, "reasoning_tokens": 13158836, "cache_read_tokens": 2489516469, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 147.8, "avg_query_seconds": 9.9, "median_query_seconds": 5.0, "avg_ttft_seconds": 2.1, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 24.242, "latency": 8712.24, "stderr": 1.515, "cost_per_test": 0.211804, "token_totals": { "input_tokens": 2175333414, "output_tokens": 16125095, "reasoning_tokens": 12753311, "cache_read_tokens": 2150748075, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 122.1, "avg_query_seconds": 49.3, "median_query_seconds": 23.2, "avg_ttft_seconds": 13.0, "output_time_split": { "reasoning_pct": 87.8, "content_pct": 1.2, "tool_call_pct": 11.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 19.697, "latency": 6427.575, "stderr": 2.314, "cost_per_test": 8.64749, "token_totals": { "input_tokens": 2400090539, "output_tokens": 12571537, "reasoning_tokens": 7741941, "cache_read_tokens": 2313842969, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 174.7, "avg_query_seconds": 6.1, "median_query_seconds": 4.5, "avg_ttft_seconds": 3.6, "output_time_split": { "reasoning_pct": 63.7, "content_pct": 34.5, "tool_call_pct": 1.7 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 17.172, "latency": 11353.52, "stderr": 0.505, "cost_per_test": 12.478176, "token_totals": { "input_tokens": 1806115861, "output_tokens": 12946019, "reasoning_tokens": 9401346, "cache_read_tokens": 1773695659, "cache_write_tokens": 31387110 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 124.1, "avg_query_seconds": 72.6, "median_query_seconds": 48.6, "avg_ttft_seconds": 27.6, "output_time_split": { "reasoning_pct": 77.6, "content_pct": 2.2, "tool_call_pct": 20.2 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 14.141, "latency": 5493.603, "stderr": 2.02, "cost_per_test": 3.311817, "token_totals": { "input_tokens": 3598466519, "output_tokens": 11740127, "reasoning_tokens": 6902591, "cache_read_tokens": 3587685589, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 232.3, "avg_query_seconds": 12.3, "median_query_seconds": 7.3, "avg_ttft_seconds": 3.3, "output_time_split": { "reasoning_pct": 75.4, "content_pct": 0.7, "tool_call_pct": 23.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 13.636, "latency": 1965.568, "stderr": 1.515, "cost_per_test": 0.354726, "token_totals": { "input_tokens": 1004162895, "output_tokens": 10957435, "reasoning_tokens": 7328334, "cache_read_tokens": 988222853, "cache_write_tokens": 15670624 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 78.9, "avg_query_seconds": 16.1, "median_query_seconds": 8.0, "avg_ttft_seconds": 6.2, "output_time_split": { "reasoning_pct": 39.2, "content_pct": 8.8, "tool_call_pct": 51.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 8.081, "latency": 7798.686, "stderr": 1.336, "cost_per_test": 3.07266, "token_totals": { "input_tokens": 2695499187, "output_tokens": 14630101, "reasoning_tokens": 10474030, "cache_read_tokens": 2628586667, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 154.5, "avg_query_seconds": 32.3, "median_query_seconds": 16.4, "avg_ttft_seconds": 6.6, "output_time_split": { "reasoning_pct": 78.0, "content_pct": 0.5, "tool_call_pct": 21.5 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse-spark-open": { "accuracy": 6.566, "latency": 7466.403, "stderr": 0.505, "cost_per_test": 7.264062, "token_totals": { "input_tokens": 916170962, "output_tokens": 7264611, "reasoning_tokens": 4188707, "cache_read_tokens": 633327467, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 90.0, "avg_query_seconds": 71.5, "median_query_seconds": 26.6, "avg_ttft_seconds": 17.7, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 4.04, "latency": 1154.552, "stderr": 1.336, "cost_per_test": null, "token_totals": { "input_tokens": 805547405, "output_tokens": 4342340, "reasoning_tokens": 2681354, "cache_read_tokens": 789244544, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 137.2, "avg_query_seconds": 1.6, "median_query_seconds": 0.8, "avg_ttft_seconds": 0.5, "output_time_split": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "inception/mercury-3": { "accuracy": 3.03, "latency": 1987.184, "stderr": 0.0, "cost_per_test": 0.421803, "token_totals": { "input_tokens": 1240859751, "output_tokens": 8811077, "reasoning_tokens": 8328363, "cache_read_tokens": 1103676197, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 136.2, "avg_query_seconds": 4.1, "median_query_seconds": 2.6, "avg_ttft_seconds": 3.4, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.505, "latency": 1378.754, "stderr": 0.505, "cost_per_test": 2.495998, "token_totals": { "input_tokens": 781467247, "output_tokens": 5735234, "reasoning_tokens": null, "cache_read_tokens": 771035008, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 140.2, "avg_query_seconds": 7.2, "median_query_seconds": 3.5, "avg_ttft_seconds": 2.5, "output_time_split": { "reasoning_pct": 99.9, "content_pct": 0.0, "tool_call_pct": 0.1 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 705.115, "stderr": 0.0, "cost_per_test": 0.321679, "token_totals": { "input_tokens": 258225964, "output_tokens": 3667960, "reasoning_tokens": null, "cache_read_tokens": 252660395, "cache_write_tokens": 0 }, "token_tasks_measured": 66, "turn_metrics": { "tasks_measured": 66, "avg_turns": 91.0, "avg_query_seconds": 5.1, "median_query_seconds": 2.7, "avg_ttft_seconds": 2.2, "output_time_split": { "reasoning_pct": 99.9, "content_pct": 0.0, "tool_call_pct": 0.1 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "tax_agent_bench": { "alibaba/qwen3.8-max-internal": { "accuracy": 49.663, "latency": 5231.239, "stderr": 3.603, "cost_per_test": 4.068548, "token_totals": { "input_tokens": 1079323232, "output_tokens": 52945750, "reasoning_tokens": 44742935, "cache_read_tokens": 987261824, "cache_write_tokens": null }, "token_tasks_measured": 184, "turn_metrics": { "tasks_measured": 184, "avg_turns": 46.3, "avg_query_seconds": 113.0, "median_query_seconds": 39.1, "avg_ttft_seconds": 12.3, "output_time_split": { "reasoning_pct": 83.1, "content_pct": 0.1, "tool_call_pct": 16.8 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 45.573, "latency": 4012.548, "stderr": 3.593, "cost_per_test": 15.086184, "token_totals": { "input_tokens": 815304119, "output_tokens": 88000495, "reasoning_tokens": 76703031, "cache_read_tokens": 648981098, "cache_write_tokens": 166280701 }, "token_tasks_measured": 190, "turn_metrics": { "tasks_measured": 190, "avg_turns": 43.1, "avg_query_seconds": 93.0, "median_query_seconds": 8.4, "avg_ttft_seconds": 13.3, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 44.56, "latency": 985.634, "stderr": 3.587, "cost_per_test": 3.473645, "token_totals": { "input_tokens": 285519334, "output_tokens": 14493886, "reasoning_tokens": 6146392, "cache_read_tokens": 200405700, "cache_write_tokens": 0 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 18.8, "avg_query_seconds": 52.5, "median_query_seconds": 13.8, "avg_ttft_seconds": 7.2, "output_time_split": { "reasoning_pct": 83.3, "content_pct": 7.5, "tool_call_pct": 9.2 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 44.35, "latency": 1264.434, "stderr": 3.574, "cost_per_test": 0.459454, "token_totals": { "input_tokens": 779160448, "output_tokens": 53561725, "reasoning_tokens": 45716077, "cache_read_tokens": 712066688, "cache_write_tokens": 0 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 41.4, "avg_query_seconds": 30.5, "median_query_seconds": 8.7, "avg_ttft_seconds": 1.5, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder-full": { "accuracy": 42.997, "latency": 1084.296, "stderr": 3.572, "cost_per_test": 2.147235, "token_totals": { "input_tokens": 257606995, "output_tokens": 9968530, "reasoning_tokens": 6086406, "cache_read_tokens": 214574250, "cache_write_tokens": 0 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 24.0, "avg_query_seconds": 45.3, "median_query_seconds": 13.4, "avg_ttft_seconds": 14.5, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 41.959, "latency": 3560.542, "stderr": 3.561, "cost_per_test": 9.245466, "token_totals": { "input_tokens": 1150865417, "output_tokens": 108029684, "reasoning_tokens": 94199819, "cache_read_tokens": 954147139, "cache_write_tokens": 196666034 }, "token_tasks_measured": 191, "turn_metrics": { "tasks_measured": 191, "avg_turns": 46.9, "avg_query_seconds": 75.8, "median_query_seconds": 15.1, "avg_ttft_seconds": 12.7, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 39.698, "latency": 3052.932, "stderr": 3.52, "cost_per_test": 1.80097, "token_totals": { "input_tokens": 229302538, "output_tokens": 47417212, "reasoning_tokens": 1131, "cache_read_tokens": 159712377, "cache_write_tokens": 0 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 11.9, "avg_query_seconds": 255.6, "median_query_seconds": 139.7, "avg_ttft_seconds": 2.7, "output_time_split": { "reasoning_pct": 86.0, "content_pct": 0.7, "tool_call_pct": 13.2 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse-wonder": { "accuracy": 36.269, "latency": 492.843, "stderr": 3.47, "cost_per_test": 1.884439, "token_totals": { "input_tokens": 225535207, "output_tokens": 8770631, "reasoning_tokens": 5466021, "cache_read_tokens": 187857044, "cache_write_tokens": 0 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 22.6, "avg_query_seconds": 21.8, "median_query_seconds": 7.3, "avg_ttft_seconds": 7.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 34.197, "latency": 996.469, "stderr": 3.423, "cost_per_test": 1.796539, "token_totals": { "input_tokens": 280409764, "output_tokens": 10821528, "reasoning_tokens": 6708987, "cache_read_tokens": 186011136, "cache_write_tokens": 0 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 27.4, "avg_query_seconds": 36.3, "median_query_seconds": 13.5, "avg_ttft_seconds": 4.5, "output_time_split": { "reasoning_pct": 74.8, "content_pct": 17.6, "tool_call_pct": 7.6 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 33.368, "latency": 2415.159, "stderr": 3.386, "cost_per_test": 2.790127, "token_totals": { "input_tokens": 444117224, "output_tokens": 14995598, "reasoning_tokens": 10782746, "cache_read_tokens": 381463574, "cache_write_tokens": 0 }, "token_tasks_measured": 189, "turn_metrics": { "tasks_measured": 189, "avg_turns": 22.4, "avg_query_seconds": 108.0, "median_query_seconds": 36.9, "avg_ttft_seconds": 8.8, "output_time_split": { "reasoning_pct": 60.8, "content_pct": 4.0, "tool_call_pct": 35.3 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 32.617, "latency": 2464.028, "stderr": 3.381, "cost_per_test": 0.585277, "token_totals": { "input_tokens": 181603577, "output_tokens": 28903691, "reasoning_tokens": 24008954, "cache_read_tokens": 139882560, "cache_write_tokens": 0 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 19.3, "avg_query_seconds": 127.6, "median_query_seconds": 39.8, "avg_ttft_seconds": 3.5, "output_time_split": { "reasoning_pct": 72.5, "content_pct": 10.7, "tool_call_pct": 16.8 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 32.388, "latency": 1547.695, "stderr": 3.361, "cost_per_test": 0.134306, "token_totals": { "input_tokens": 136588210, "output_tokens": 12521886, "reasoning_tokens": 8284595, "cache_read_tokens": 102894848, "cache_write_tokens": 0 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 12.7, "avg_query_seconds": 121.8, "median_query_seconds": 75.6, "avg_ttft_seconds": 5.3, "output_time_split": { "reasoning_pct": 80.3, "content_pct": 5.4, "tool_call_pct": 14.3 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 32.136, "latency": 1962.771, "stderr": 3.342, "cost_per_test": 0.320948, "token_totals": { "input_tokens": 551883325, "output_tokens": 70312257, "reasoning_tokens": 62154212, "cache_read_tokens": 466865454, "cache_write_tokens": 0 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 31.1, "avg_query_seconds": 63.2, "median_query_seconds": 19.4, "avg_ttft_seconds": 1.1, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 32.124, "latency": 178.329, "stderr": 3.37, "cost_per_test": 0.857214, "token_totals": { "input_tokens": 262586790, "output_tokens": 7012701, "reasoning_tokens": 4789793, "cache_read_tokens": 208172664, "cache_write_tokens": 0 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 30.4, "avg_query_seconds": 5.9, "median_query_seconds": 2.7, "avg_ttft_seconds": 2.5, "output_time_split": { "reasoning_pct": 98.9, "content_pct": 0.2, "tool_call_pct": 0.9 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 31.922, "latency": 2973.647, "stderr": 3.352, "cost_per_test": 1.164207, "token_totals": { "input_tokens": 782193467, "output_tokens": 143162719, "reasoning_tokens": 133330984, "cache_read_tokens": 613966554, "cache_write_tokens": 168173092 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 33.6, "avg_query_seconds": 88.5, "median_query_seconds": 26.2, "avg_ttft_seconds": 15.5, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 31.55, "latency": 1297.006, "stderr": 3.35, "cost_per_test": 0.138062, "token_totals": { "input_tokens": 194143568, "output_tokens": 28577937, "reasoning_tokens": 24325230, "cache_read_tokens": 139704320, "cache_write_tokens": 0 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 14.4, "avg_query_seconds": 90.3, "median_query_seconds": 45.4, "avg_ttft_seconds": 2.9, "output_time_split": { "reasoning_pct": 84.4, "content_pct": 1.6, "tool_call_pct": 14.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 28.507, "latency": 1038.928, "stderr": 3.14, "cost_per_test": 0.048982, "token_totals": { "input_tokens": 168296091, "output_tokens": 12132305, "reasoning_tokens": 7352234, "cache_read_tokens": 127944128, "cache_write_tokens": 0 }, "token_tasks_measured": 192, "turn_metrics": { "tasks_measured": 192, "avg_turns": 15.2, "avg_query_seconds": 68.3, "median_query_seconds": 36.2, "avg_ttft_seconds": 4.6, "output_time_split": { "reasoning_pct": 2.4, "content_pct": 0.1, "tool_call_pct": 97.6 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 26.756, "latency": 1596.755, "stderr": 3.093, "cost_per_test": 0.729136, "token_totals": { "input_tokens": 172298347, "output_tokens": 21059828, "reasoning_tokens": 17608474, "cache_read_tokens": 133313152, "cache_write_tokens": 0 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 12.9, "avg_query_seconds": 123.5, "median_query_seconds": 57.8, "avg_ttft_seconds": 1.7, "output_time_split": { "reasoning_pct": 82.5, "content_pct": 2.2, "tool_call_pct": 15.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 26.321, "latency": 701.825, "stderr": 2.653, "cost_per_test": 0.883559, "token_totals": { "input_tokens": 413690950, "output_tokens": 3571385, "reasoning_tokens": 1667757, "cache_read_tokens": 329680688, "cache_write_tokens": 0 }, "token_tasks_measured": 192, "turn_metrics": { "tasks_measured": 192, "avg_turns": 15.0, "avg_query_seconds": 46.8, "median_query_seconds": 11.3, "avg_ttft_seconds": 7.2, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 24.87, "latency": 3518.57, "stderr": 3.12, "cost_per_test": 3.643476, "token_totals": { "input_tokens": 393567716, "output_tokens": 32589454, "reasoning_tokens": 26739791, "cache_read_tokens": 251978240, "cache_write_tokens": 140846717 }, "token_tasks_measured": 181, "turn_metrics": { "tasks_measured": 181, "avg_turns": 20.7, "avg_query_seconds": 169.9, "median_query_seconds": 36.7, "avg_ttft_seconds": 6.8, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 21.754, "latency": 1213.878, "stderr": 2.977, "cost_per_test": 1.323601, "token_totals": { "input_tokens": 149308416, "output_tokens": 12717430, "reasoning_tokens": 9488118, "cache_read_tokens": 103439338, "cache_write_tokens": 45805894 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 11.9, "avg_query_seconds": 102.3, "median_query_seconds": 31.3, "avg_ttft_seconds": 19.6, "output_time_split": { "reasoning_pct": 17.4, "content_pct": 0.2, "tool_call_pct": 82.4 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 20.717, "latency": 900.586, "stderr": 2.924, "cost_per_test": 5.7963, "token_totals": { "input_tokens": 115812555, "output_tokens": 12032734, "reasoning_tokens": 9165100, "cache_read_tokens": 71528751, "cache_write_tokens": null }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 10.2, "avg_query_seconds": 88.5, "median_query_seconds": 31.0, "avg_ttft_seconds": 14.3, "output_time_split": { "reasoning_pct": 20.8, "content_pct": 0.4, "tool_call_pct": 78.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 19.947, "latency": 1995.736, "stderr": 2.866, "cost_per_test": 0.190956, "token_totals": { "input_tokens": 449103207, "output_tokens": 40664272, "reasoning_tokens": 37199155, "cache_read_tokens": 368052701, "cache_write_tokens": 80918209 }, "token_tasks_measured": 190, "turn_metrics": { "tasks_measured": 190, "avg_turns": 23.9, "avg_query_seconds": 83.4, "median_query_seconds": 18.0, "avg_ttft_seconds": 6.5, "output_time_split": { "reasoning_pct": 77.3, "content_pct": 0.1, "tool_call_pct": 22.7 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-3": { "accuracy": 18.169, "latency": 269.272, "stderr": 2.536, "cost_per_test": 0.045925, "token_totals": { "input_tokens": 107781718, "output_tokens": 14189123, "reasoning_tokens": 12634168, "cache_read_tokens": 76454803, "cache_write_tokens": 0 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 13.4, "avg_query_seconds": 20.1, "median_query_seconds": 7.2, "avg_ttft_seconds": 5.4, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 15.389, "latency": 385.252, "stderr": 2.365, "cost_per_test": 0.091767, "token_totals": { "input_tokens": 107926904, "output_tokens": 7430357, "reasoning_tokens": null, "cache_read_tokens": 98264704, "cache_write_tokens": 0 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 26.1, "avg_query_seconds": 14.8, "median_query_seconds": 3.5, "avg_ttft_seconds": 3.1, "output_time_split": { "reasoning_pct": 99.9, "content_pct": 0.0, "tool_call_pct": 0.1 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 15.291, "latency": 810.557, "stderr": 2.423, "cost_per_test": 0.353757, "token_totals": { "input_tokens": 147459631, "output_tokens": 7269937, "reasoning_tokens": null, "cache_read_tokens": 130876928, "cache_write_tokens": 0 }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 24.1, "avg_query_seconds": 33.7, "median_query_seconds": 6.4, "avg_ttft_seconds": 2.4, "output_time_split": { "reasoning_pct": 99.9, "content_pct": 0.0, "tool_call_pct": 0.1 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 15.166, "latency": 2751.35, "stderr": 2.426, "cost_per_test": 2.159819, "token_totals": { "input_tokens": 601151954, "output_tokens": 45185676, "reasoning_tokens": 40035547, "cache_read_tokens": 486107389, "cache_write_tokens": 114824859 }, "token_tasks_measured": 189, "turn_metrics": { "tasks_measured": 189, "avg_turns": 22.2, "avg_query_seconds": 123.7, "median_query_seconds": 20.7, "avg_ttft_seconds": 8.5, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 8.808, "latency": 57.987, "stderr": 1.902, "cost_per_test": null, "token_totals": { "input_tokens": 110261613, "output_tokens": 4023578, "reasoning_tokens": 2422138, "cache_read_tokens": 88299456, "cache_write_tokens": null }, "token_tasks_measured": 193, "turn_metrics": { "tasks_measured": 193, "avg_turns": 19.2, "avg_query_seconds": 3.0, "median_query_seconds": 1.3, "avg_ttft_seconds": 0.4, "output_time_split": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null } } } }