{ "metadata": { "benchmark": "Vals Index", "slug": "vals_index", "description": "A single measure of AI's potential economic impact \u2014 agentic model performance across finance, coding, legal, and tax tasks, weighted by each sector's share of U.S. GDP.", "benchmark_id": "vals_index", "family": "vals_index", "version": "2.1", "updated": "2026-10-07", "dataset_type": "private", "industry": "index", "tasks": { "overall": "Overall", "finance_agent": "Finance Agent v2", "emb": "EMB", "terminal_bench_4": "Terminal-Bench 4.0", "vibe_code_bench": "Vibe Code Bench", "code_migration": "Code Migration", "legal_research": "Legal Research", "legal_agent_benchmark": "HLAB", "tax_agent_bench": "Tax Agent" }, "models": [ "alibaba/qwen3.8-max", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "fireworks/ember-1", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-4.7", "inception/mercury-2.5", "kimi/kimi-k3", "meta/muse_spark_1_2", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "minimax/MiniMax-M3", "mistralai/mistral-large-4", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "stepfun/step-5-preview", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.3" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 45 }, "tasks": { "overall": { "google/gemini-4-argon": { "accuracy": 68.896, "latency": 2792.769, "stderr": 0.974, "cost_per_test": 15.681703, "token_totals": { "input_tokens": 13395185446, "output_tokens": 142006483, "reasoning_tokens": 58820461, "cache_read_tokens": 11649077653, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 67.037, "latency": 4684.303, "stderr": 0.92, "cost_per_test": 21.344634, "token_totals": { "input_tokens": 40376654627, "output_tokens": 598676506, "reasoning_tokens": null, "cache_read_tokens": 38748235131, "cache_write_tokens": 1628023492 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 66.972, "latency": 4740.124, "stderr": 0.888, "cost_per_test": 32.143857, "token_totals": { "input_tokens": 28569832018, "output_tokens": 477677247, "reasoning_tokens": null, "cache_read_tokens": 27292436406, "cache_write_tokens": 1277102516 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 65.827, "latency": 4611.975, "stderr": 1.115, "cost_per_test": 28.711785, "token_totals": { "input_tokens": 18942153740, "output_tokens": 202215998, "reasoning_tokens": null, "cache_read_tokens": 16623792412, "cache_write_tokens": 2318124479 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 63.674, "latency": 3461.558, "stderr": 0.957, "cost_per_test": 19.314752, "token_totals": { "input_tokens": 14709112797, "output_tokens": 146953550, "reasoning_tokens": null, "cache_read_tokens": 13967647934, "cache_write_tokens": 741228784 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 63.125, "latency": 1667.737, "stderr": 1.19, "cost_per_test": 18.457813, "token_totals": { "input_tokens": 3926682957, "output_tokens": 94522123, "reasoning_tokens": 58991004, "cache_read_tokens": 3435837939, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 61.388, "latency": 2544.13, "stderr": 0.999, "cost_per_test": 29.58529, "token_totals": { "input_tokens": 10427208643, "output_tokens": 124294394, "reasoning_tokens": null, "cache_read_tokens": 9952899434, "cache_write_tokens": 474114355 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 61.154, "latency": 2590.007, "stderr": 1.011, "cost_per_test": 3.236975, "token_totals": { "input_tokens": 4496496495, "output_tokens": 104770923, "reasoning_tokens": 66885965, "cache_read_tokens": 4094274123, "cache_write_tokens": 401029150 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 58.162, "latency": 1414.241, "stderr": 1.188, "cost_per_test": 3.787017, "token_totals": { "input_tokens": 11276923364, "output_tokens": 94121662, "reasoning_tokens": 47554265, "cache_read_tokens": 10840632150, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 58.005, "latency": 2242.736, "stderr": 1.017, "cost_per_test": 14.239384, "token_totals": { "input_tokens": 6458937700, "output_tokens": 145311459, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 57.536, "latency": 1750.678, "stderr": 1.012, "cost_per_test": 7.580816, "token_totals": { "input_tokens": 8969500048, "output_tokens": 104585296, "reasoning_tokens": 73395716, "cache_read_tokens": 8455096187, "cache_write_tokens": 512773748 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 55.199, "latency": 4038.85, "stderr": 1.182, "cost_per_test": 0.410427, "token_totals": { "input_tokens": 8348954316, "output_tokens": 117823035, "reasoning_tokens": 78548135, "cache_read_tokens": 7874523050, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 55.1, "latency": 2388.428, "stderr": 1.009, "cost_per_test": 13.136067, "token_totals": { "input_tokens": 7983128075, "output_tokens": 118224562, "reasoning_tokens": null, "cache_read_tokens": 7526309059, "cache_write_tokens": 456581307 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 54.947, "latency": 2131.435, "stderr": 1.072, "cost_per_test": 12.118089, "token_totals": { "input_tokens": 7281370578, "output_tokens": 114000299, "reasoning_tokens": 66410523, "cache_read_tokens": 6279335339, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 54.825, "latency": 3117.158, "stderr": 1.028, "cost_per_test": 5.728779, "token_totals": { "input_tokens": 13010993484, "output_tokens": 114565688, "reasoning_tokens": 66858568, "cache_read_tokens": 11766322656, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 54.312, "latency": 2836.532, "stderr": 1.41, "cost_per_test": 2.991465, "token_totals": { "input_tokens": 21932670544, "output_tokens": 747949485, "reasoning_tokens": null, "cache_read_tokens": 20548020147, "cache_write_tokens": 1384333211 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 53.514, "latency": 4392.771, "stderr": 1.305, "cost_per_test": 7.248682, "token_totals": { "input_tokens": 12086239659, "output_tokens": 211635329, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 53.231, "latency": 3481.728, "stderr": 1.097, "cost_per_test": 0.202364, "token_totals": { "input_tokens": 11537436670, "output_tokens": 124572549, "reasoning_tokens": 73711922, "cache_read_tokens": 10948185515, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 53.197, "latency": 1965.589, "stderr": 1.066, "cost_per_test": 3.37642, "token_totals": { "input_tokens": 8123523682, "output_tokens": 77279392, "reasoning_tokens": 39625354, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 53.085, "latency": 2138.853, "stderr": 1.293, "cost_per_test": 5.776241, "token_totals": { "input_tokens": 7638080888, "output_tokens": 231741382, "reasoning_tokens": 188924288, "cache_read_tokens": 6809323948, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 52.092, "latency": 2221.07, "stderr": 1.153, "cost_per_test": 4.49288, "token_totals": { "input_tokens": 4418649802, "output_tokens": 103260599, "reasoning_tokens": 56689713, "cache_read_tokens": 3928143616, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 51.775, "latency": 3255.608, "stderr": 1.092, "cost_per_test": 13.721168, "token_totals": { "input_tokens": 26641890343, "output_tokens": 166758155, "reasoning_tokens": null, "cache_read_tokens": 26149187593, "cache_write_tokens": 492339900 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 51.687, "latency": 1767.438, "stderr": 1.057, "cost_per_test": 0.824394, "token_totals": { "input_tokens": 9483346035, "output_tokens": 226963302, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 51.32, "latency": 1673.173, "stderr": 1.13, "cost_per_test": 0.332286, "token_totals": { "input_tokens": 10053492190, "output_tokens": 138822730, "reasoning_tokens": 104138507, "cache_read_tokens": 9747009189, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 51.273, "latency": 2847.08, "stderr": 1.097, "cost_per_test": 4.708904, "token_totals": { "input_tokens": 9195740143, "output_tokens": 89094712, "reasoning_tokens": 36815592, "cache_read_tokens": 8138312568, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 51.216, "latency": 1833.681, "stderr": 1.072, "cost_per_test": 0.431072, "token_totals": { "input_tokens": 9742841837, "output_tokens": 201054852, "reasoning_tokens": 166505433, "cache_read_tokens": 9215330223, "cache_write_tokens": 525919863 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/ember-1": { "accuracy": 50.811, "latency": 4261.402, "stderr": 1.277, "cost_per_test": 9.577707, "token_totals": { "input_tokens": 12817257164, "output_tokens": 95671120, "reasoning_tokens": 54007808, "cache_read_tokens": 12255356080, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 50.297, "latency": 4103.017, "stderr": 0.992, "cost_per_test": 6.382152, "token_totals": { "input_tokens": 7805993994, "output_tokens": 86191753, "reasoning_tokens": 52055744, "cache_read_tokens": 7427476650, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 49.942, "latency": 3656.53, "stderr": 1.149, "cost_per_test": 1.438575, "token_totals": { "input_tokens": 10714625549, "output_tokens": 179905442, "reasoning_tokens": 134915925, "cache_read_tokens": 10226411584, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "stepfun/step-5-preview": { "accuracy": 49.354, "latency": 4217.914, "stderr": 1.222, "cost_per_test": 2.625676, "token_totals": { "input_tokens": 16891548436, "output_tokens": 132962363, "reasoning_tokens": 0, "cache_read_tokens": 16257538390, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 49.287, "latency": 1376.299, "stderr": 1.096, "cost_per_test": 1.942052, "token_totals": { "input_tokens": 5061541650, "output_tokens": 63062770, "reasoning_tokens": 25481198, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 48.27, "latency": 6070.722, "stderr": 1.337, "cost_per_test": 4.964352, "token_totals": { "input_tokens": 7745942147, "output_tokens": 148005514, "reasoning_tokens": 109384504, "cache_read_tokens": 7241866284, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 48.045, "latency": 6350.407, "stderr": 1.112, "cost_per_test": 13.776396, "token_totals": { "input_tokens": 11219473063, "output_tokens": 166375436, "reasoning_tokens": null, "cache_read_tokens": 6156054186, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 47.963, "latency": 2585.953, "stderr": 1.24, "cost_per_test": 0.847519, "token_totals": { "input_tokens": 14496126536, "output_tokens": 131103564, "reasoning_tokens": 93375183, "cache_read_tokens": 13888877777, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 47.628, "latency": 3845.238, "stderr": 1.116, "cost_per_test": 3.352436, "token_totals": { "input_tokens": 18051799594, "output_tokens": 152969113, "reasoning_tokens": 107303370, "cache_read_tokens": 17128953984, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 44.786, "latency": 1097.194, "stderr": 1.098, "cost_per_test": 3.297245, "token_totals": { "input_tokens": 6901669422, "output_tokens": 86206477, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 44.723, "latency": 944.195, "stderr": 1.171, "cost_per_test": 2.458898, "token_totals": { "input_tokens": 2285178801, "output_tokens": 55497604, "reasoning_tokens": 16660782, "cache_read_tokens": 1961973829, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 38.63, "latency": 1928.908, "stderr": 1.157, "cost_per_test": 1.30566, "token_totals": { "input_tokens": 8061319875, "output_tokens": 94395896, "reasoning_tokens": 57723581, "cache_read_tokens": 7702599731, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 36.535, "latency": 2334.674, "stderr": 1.167, "cost_per_test": 3.082399, "token_totals": { "input_tokens": 9639172740, "output_tokens": 70307024, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 33.842, "latency": 2073.112, "stderr": 1.171, "cost_per_test": 0.161806, "token_totals": { "input_tokens": 3114510241, "output_tokens": 46631531, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 33.442, "latency": 591.992, "stderr": 1.133, "cost_per_test": 2.180322, "token_totals": { "input_tokens": 3121346524, "output_tokens": 27309809, "reasoning_tokens": 16027963, "cache_read_tokens": 2662909894, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 33.169, "latency": 1788.542, "stderr": 1.282, "cost_per_test": 1.45173, "token_totals": { "input_tokens": 5512390012, "output_tokens": 219610335, "reasoning_tokens": 197363641, "cache_read_tokens": 4699214805, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 28.668, "latency": 1205.929, "stderr": 1.1, "cost_per_test": 1.464372, "token_totals": { "input_tokens": 4513289654, "output_tokens": 69323896, "reasoning_tokens": null, "cache_read_tokens": 4170701798, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 25.459, "latency": 1261.927, "stderr": 1.032, "cost_per_test": 0.281793, "token_totals": { "input_tokens": 2529140240, "output_tokens": 67043835, "reasoning_tokens": null, "cache_read_tokens": 2348746454, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-2.5": { "accuracy": 8.855, "latency": 341.744, "stderr": 0.446, "cost_per_test": 0.285616, "token_totals": { "input_tokens": 1951322288, "output_tokens": 23101371, "reasoning_tokens": 20973725, "cache_read_tokens": 1075849375, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "vibe_code_bench": { "anthropic/claude-sonnet-5-5": { "accuracy": 92.392, "latency": 3672.106, "stderr": 1.26, "cost_per_test": 31.250578, "token_totals": { "input_tokens": 5822927217, "output_tokens": 25491041, "reasoning_tokens": 11893204, "cache_read_tokens": 5760733058, "cache_write_tokens": 62167153 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "google/gemini-4-argon": { "accuracy": 91.906, "latency": 2247.675, "stderr": 1.899, "cost_per_test": 8.10797, "token_totals": { "input_tokens": 451864686, "output_tokens": 5436370, "reasoning_tokens": 849287, "cache_read_tokens": 397575696, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "anthropic/claude-haiku-5-5": { "accuracy": 90.444, "latency": 2075.723, "stderr": 1.725, "cost_per_test": 6.071856, "token_totals": { "input_tokens": 4600663825, "output_tokens": 23863970, "reasoning_tokens": 13235182, "cache_read_tokens": 4559642564, "cache_write_tokens": 40996929 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "anthropic/claude-fable-5": { "accuracy": 90.352, "latency": 3711.026, "stderr": 2.097, "cost_per_test": 41.714694, "token_totals": { "input_tokens": 1628443000, "output_tokens": 8412174, "reasoning_tokens": 2231064, "cache_read_tokens": 1604184760, "cache_write_tokens": 24245069 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "anthropic/claude-opus-5-5": { "accuracy": 90.294, "latency": 5766.524, "stderr": 1.526, "cost_per_test": 57.922811, "token_totals": { "input_tokens": 5596583425, "output_tokens": 19722501, "reasoning_tokens": 7457724, "cache_read_tokens": 5539450465, "cache_write_tokens": 57102136 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "anthropic/claude-fable-5-1": { "accuracy": 90.263, "latency": 3459.994, "stderr": 1.566, "cost_per_test": 33.367507, "token_totals": { "input_tokens": 2858376432, "output_tokens": 11229337, "reasoning_tokens": 2013796, "cache_read_tokens": 2826345790, "cache_write_tokens": 32006248 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "openai/gpt-6-astra": { "accuracy": 89.594, "latency": 2344.127, "stderr": 2.174, "cost_per_test": 38.512323, "token_totals": { "input_tokens": 813962842, "output_tokens": 7672087, "reasoning_tokens": 3013580, "cache_read_tokens": 762745769, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "openai/gpt-6.1-sol": { "accuracy": 88.931, "latency": 4489.755, "stderr": 1.893, "cost_per_test": 6.234794, "token_totals": { "input_tokens": 1073502468, "output_tokens": 10186788, "reasoning_tokens": 5383848, "cache_read_tokens": 1056860976, "cache_write_tokens": 16623336 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "anthropic/claude-opus-5": { "accuracy": 88.404, "latency": 5301.22, "stderr": 3.001, "cost_per_test": 33.876235, "token_totals": { "input_tokens": 2505984639, "output_tokens": 9909555, "reasoning_tokens": 1547105, "cache_read_tokens": 2470549366, "cache_write_tokens": 35417005 }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "OpenHands" }, "openai/gpt-6-sol": { "accuracy": 87.824, "latency": 2336.393, "stderr": 2.526, "cost_per_test": 26.356012, "token_totals": { "input_tokens": 3192235437, "output_tokens": 6823472, "reasoning_tokens": 2395580, "cache_read_tokens": 3162891655, "cache_write_tokens": 29258099 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "grok/grok-4.7": { "accuracy": 86.175, "latency": 2147.065, "stderr": 2.181, "cost_per_test": 15.82706, "token_totals": { "input_tokens": 767137889, "output_tokens": 7923865, "reasoning_tokens": 3838088, "cache_read_tokens": 713123456, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "meta/muse_spark_1_3_max": { "accuracy": 85.856, "latency": 977.668, "stderr": 2.508, "cost_per_test": 2.543084, "token_totals": { "input_tokens": 651739493, "output_tokens": 3048548, "reasoning_tokens": 422854, "cache_read_tokens": 636796800, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "xiaomi/mimo-v2.6-pro": { "accuracy": 85.223, "latency": 3646.574, "stderr": 3.392, "cost_per_test": 1.043846, "token_totals": { "input_tokens": 1076595168, "output_tokens": 7478091, "reasoning_tokens": 2852217, "cache_read_tokens": 979676672, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands" }, "kimi/kimi-k3": { "accuracy": 84.968, "latency": 999.021, "stderr": 2.746, "cost_per_test": 10.007576, "token_totals": { "input_tokens": 1261633418, "output_tokens": 5010801, "reasoning_tokens": 1248736, "cache_read_tokens": 1244327217, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "OpenHands" }, "deepseek/deepseek-v4.1-flash": { "accuracy": 84.739, "latency": 928.101, "stderr": 2.891, "cost_per_test": 0.407445, "token_totals": { "input_tokens": 1429431621, "output_tokens": 7186472, "reasoning_tokens": 1913924, "cache_read_tokens": 1418642812, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "fireworks/ember-1": { "accuracy": 83.869, "latency": 2357.968, "stderr": 3.151, "cost_per_test": 8.559507, "token_totals": { "input_tokens": 981791343, "output_tokens": 4071290, "reasoning_tokens": 774425, "cache_read_tokens": 954988156, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "OpenHands" }, "meta/muse_spark_1_3": { "accuracy": 82.859, "latency": 773.797, "stderr": 2.901, "cost_per_test": 2.101762, "token_totals": { "input_tokens": 401837405, "output_tokens": 2713902, "reasoning_tokens": 370079, "cache_read_tokens": 371584301, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "anthropic/claude-opus-4-8": { "accuracy": 82.725, "latency": 4549.256, "stderr": 3.081, "cost_per_test": 26.879989, "token_totals": { "input_tokens": 1802936139, "output_tokens": 9844882, "reasoning_tokens": 3416693, "cache_read_tokens": 1781691366, "cache_write_tokens": 21227618 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 82.297, "latency": 3982.6, "stderr": 3.148, "cost_per_test": 0.356023, "token_totals": { "input_tokens": 1830028483, "output_tokens": 7746431, "reasoning_tokens": 2336919, "cache_read_tokens": 1820296320, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "openai/gpt-6-luna": { "accuracy": 81.649, "latency": 2219.515, "stderr": 3.377, "cost_per_test": 1.345703, "token_totals": { "input_tokens": 3186713261, "output_tokens": 9004550, "reasoning_tokens": 3632878, "cache_read_tokens": 3158117507, "cache_write_tokens": 28566327 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "anthropic/claude-sonnet-5": { "accuracy": 81.325, "latency": 4001.424, "stderr": 3.046, "cost_per_test": 25.38542, "token_totals": { "input_tokens": 5886886074, "output_tokens": 13734667, "reasoning_tokens": 5002326, "cache_read_tokens": 5849205195, "cache_write_tokens": 37648445 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "openai/gpt-5.6-sol": { "accuracy": 80.495, "latency": 2033.047, "stderr": 3.716, "cost_per_test": 33.403634, "token_totals": { "input_tokens": 85484634, "output_tokens": 3878003, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "meta/muse_spark_1_2": { "accuracy": 79.1, "latency": 1195.221, "stderr": 3.305, "cost_per_test": 1.529642, "token_totals": { "input_tokens": 342866386, "output_tokens": 2351182, "reasoning_tokens": 239369, "cache_read_tokens": 329175836, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "xiaomi/mimo-v2.6-flash": { "accuracy": 78.96, "latency": 3013.012, "stderr": 4.045, "cost_per_test": 0.563041, "token_totals": { "input_tokens": 1311689878, "output_tokens": 7947370, "reasoning_tokens": 3093850, "cache_read_tokens": 1157695808, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands" }, "google/gemini-3.8-flash": { "accuracy": 78.651, "latency": 519.104, "stderr": 3.884, "cost_per_test": 6.865141, "token_totals": { "input_tokens": 1149521706, "output_tokens": 6144190, "reasoning_tokens": 1439332, "cache_read_tokens": 1057116240, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "mistralai/mistral-large-4": { "accuracy": 78.403, "latency": 10395.145, "stderr": 3.55, "cost_per_test": 39.456533, "token_totals": { "input_tokens": 1823486517, "output_tokens": 8065079, "reasoning_tokens": null, "cache_read_tokens": 443300864, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": "OpenHands" }, "zai/glm-5.3": { "accuracy": 78.125, "latency": 3847.197, "stderr": 3.94, "cost_per_test": 12.450185, "token_totals": { "input_tokens": 1971471587, "output_tokens": 8173329, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "tencent/hy4-preview": { "accuracy": 77.478, "latency": 2493.104, "stderr": 4.035, "cost_per_test": 2.181516, "token_totals": { "input_tokens": 1648623995, "output_tokens": 6263219, "reasoning_tokens": 1671679, "cache_read_tokens": 1618107200, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": "OpenHands" }, "openai/gpt-5.6-luna": { "accuracy": 77.055, "latency": 1551.831, "stderr": 3.069, "cost_per_test": 0.726426, "token_totals": { "input_tokens": 54304210, "output_tokens": 3660213, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "grok/grok-4.6": { "accuracy": 76.239, "latency": 1530.145, "stderr": 3.824, "cost_per_test": 4.875513, "token_totals": { "input_tokens": 395097143, "output_tokens": 3804849, "reasoning_tokens": 775807, "cache_read_tokens": 379498496, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 74.736, "latency": 2241.572, "stderr": 3.442, "cost_per_test": 0.905592, "token_totals": { "input_tokens": 2153389585, "output_tokens": 7751888, "reasoning_tokens": 2696365, "cache_read_tokens": 2141888000, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "openai/gpt-5.6-terra": { "accuracy": 74.594, "latency": 1152.433, "stderr": 4.203, "cost_per_test": 7.887015, "token_totals": { "input_tokens": 1192383214, "output_tokens": 4865371, "reasoning_tokens": 1379403, "cache_read_tokens": 1138222288, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "stepfun/step-5-preview": { "accuracy": 70.399, "latency": 5466.358, "stderr": 4.267, "cost_per_test": 5.856257, "token_totals": { "input_tokens": 2504815838, "output_tokens": 8182685, "reasoning_tokens": 0, "cache_read_tokens": 2351680256, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": "OpenHands" }, "google/gemini-3.7-flash": { "accuracy": 70.395, "latency": 1423.494, "stderr": 4.837, "cost_per_test": 4.827643, "token_totals": { "input_tokens": 911735026, "output_tokens": 5880420, "reasoning_tokens": 1013076, "cache_read_tokens": 866906330, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "grok/grok-4.5": { "accuracy": 69.002, "latency": 830.057, "stderr": 4.48, "cost_per_test": 4.102194, "token_totals": { "input_tokens": 264749332, "output_tokens": 3594207, "reasoning_tokens": 64753, "cache_read_tokens": 254635589, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "alibaba/qwen3.8-max": { "accuracy": 64.696, "latency": 10135.139, "stderr": 5.362, "cost_per_test": 8.238852, "token_totals": { "input_tokens": 1000088567, "output_tokens": 6067236, "reasoning_tokens": 2004745, "cache_read_tokens": 921541719, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "deepseek/deepseek-v4-pro": { "accuracy": 49.931, "latency": 3418.92, "stderr": 4.767, "cost_per_test": 2.205991, "token_totals": { "input_tokens": 1767336432, "output_tokens": 7905241, "reasoning_tokens": 2425211, "cache_read_tokens": 1758006109, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "google/gemini-3.5-flash": { "accuracy": 48.683, "latency": 893.074, "stderr": 4.727, "cost_per_test": 2.540385, "token_totals": { "input_tokens": 64968703, "output_tokens": 3285132, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 47.969, "latency": 2055.117, "stderr": 5.606, "cost_per_test": 1.191299, "token_totals": { "input_tokens": 1529114850, "output_tokens": 14510964, "reasoning_tokens": 8384750, "cache_read_tokens": 1305150933, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "minimax/MiniMax-M3": { "accuracy": 47.566, "latency": 5073.23, "stderr": 5.442, "cost_per_test": 6.453899, "token_totals": { "input_tokens": 61631839, "output_tokens": 5567819, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": "OpenHands" }, "xiaomi/mimo-v2.5-pro": { "accuracy": 34.113, "latency": 2403.764, "stderr": 4.529, "cost_per_test": 0.166747, "token_totals": { "input_tokens": 7242687, "output_tokens": 3078284, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands" }, "google/gemini-3.1-pro-preview": { "accuracy": 32.034, "latency": 1211.906, "stderr": 4.342, "cost_per_test": 3.825202, "token_totals": { "input_tokens": 722761650, "output_tokens": 1941198, "reasoning_tokens": 260725, "cache_read_tokens": 678233584, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "thinkingmachines/inkling": { "accuracy": 19.214, "latency": 1105.492, "stderr": 3.659, "cost_per_test": 1.605949, "token_totals": { "input_tokens": 385384338, "output_tokens": 2166011, "reasoning_tokens": 0, "cache_read_tokens": 380152294, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": "OpenHands" }, "thinkingmachines/inkling-small": { "accuracy": 19.052, "latency": 2949.469, "stderr": 3.403, "cost_per_test": 0.432264, "token_totals": { "input_tokens": 294407624, "output_tokens": 2135670, "reasoning_tokens": null, "cache_read_tokens": 289327616, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": "OpenHands" }, "inception/mercury-2.5": { "accuracy": 3.683, "latency": 193.041, "stderr": 1.407, "cost_per_test": 0.378016, "token_totals": { "input_tokens": 198612247, "output_tokens": 1426011, "reasoning_tokens": 1298893, "cache_read_tokens": 121617600, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": "OpenHands" } }, "finance_agent": { "google/gemini-4-argon": { "accuracy": 65.401, "latency": 980.524, "stderr": 0.321, "cost_per_test": 4.378161, "token_totals": { "input_tokens": 609679241, "output_tokens": 31503850, "reasoning_tokens": 13402361, "cache_read_tokens": 289110948, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 61.435, "latency": 200.952, "stderr": 0.128, "cost_per_test": 2.000457, "token_totals": { "input_tokens": 1118010382, "output_tokens": 25909358, "reasoning_tokens": 18104089, "cache_read_tokens": 719355707, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 60.599, "latency": 304.205, "stderr": 0.279, "cost_per_test": 0.76972, "token_totals": { "input_tokens": 424942458, "output_tokens": 12218400, "reasoning_tokens": 6559929, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 59.958, "latency": 198.929, "stderr": 2.06, "cost_per_test": 0.756512, "token_totals": { "input_tokens": 508981355, "output_tokens": 16233804, "reasoning_tokens": 9783963, "cache_read_tokens": 331627422, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 59.042, "latency": 181.334, "stderr": 0.272, "cost_per_test": 1.477884, "token_totals": { "input_tokens": 711083728, "output_tokens": 16887053, "reasoning_tokens": 10058516, "cache_read_tokens": 391282104, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 58.901, "latency": 341.936, "stderr": 0.444, "cost_per_test": 0.741132, "token_totals": { "input_tokens": 413247670, "output_tokens": 14548497, "reasoning_tokens": 8843980, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 58.877, "latency": 1094.165, "stderr": 2.058, "cost_per_test": 8.346696, "token_totals": { "input_tokens": 458174812, "output_tokens": 33030168, "reasoning_tokens": 19377315, "cache_read_tokens": 228829928, "cache_write_tokens": 229316981 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 58.633, "latency": 598.424, "stderr": 0.077, "cost_per_test": 5.124473, "token_totals": { "input_tokens": 543793980, "output_tokens": 22921224, "reasoning_tokens": 9150666, "cache_read_tokens": 253099566, "cache_write_tokens": 290670709 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 58.587, "latency": 1987.81, "stderr": 0.17, "cost_per_test": 9.215734, "token_totals": { "input_tokens": 763751733, "output_tokens": 109304455, "reasoning_tokens": 94811735, "cache_read_tokens": 413590027, "cache_write_tokens": 350117722 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 58.103, "latency": 1903.336, "stderr": 0.673, "cost_per_test": 6.552431, "token_totals": { "input_tokens": 1121526513, "output_tokens": 137870326, "reasoning_tokens": 120529788, "cache_read_tokens": 555448124, "cache_write_tokens": 566027579 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 57.861, "latency": 322.279, "stderr": 0.231, "cost_per_test": 2.505488, "token_totals": { "input_tokens": 1227406624, "output_tokens": 26782944, "reasoning_tokens": 20834822, "cache_read_tokens": 707613227, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 57.339, "latency": 648.075, "stderr": 0.574, "cost_per_test": 0.198909, "token_totals": { "input_tokens": 305876104, "output_tokens": 13378019, "reasoning_tokens": 7659784, "cache_read_tokens": 127922645, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 56.314, "latency": 610.952, "stderr": 0.84, "cost_per_test": 8.061563, "token_totals": { "input_tokens": 311882539, "output_tokens": 19969523, "reasoning_tokens": null, "cache_read_tokens": 110357130, "cache_write_tokens": 201446408 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 56.277, "latency": 421.99, "stderr": 0.462, "cost_per_test": 0.071536, "token_totals": { "input_tokens": 349885254, "output_tokens": 15078803, "reasoning_tokens": 7475980, "cache_read_tokens": 154212352, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 55.84, "latency": 949.539, "stderr": 2.066, "cost_per_test": 1.072768, "token_totals": { "input_tokens": 363886559, "output_tokens": 36729670, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 55.063, "latency": 1424.632, "stderr": 0.308, "cost_per_test": 0.5903, "token_totals": { "input_tokens": 316560636, "output_tokens": 41494337, "reasoning_tokens": 34844249, "cache_read_tokens": 129727253, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 55.044, "latency": 772.153, "stderr": 0.309, "cost_per_test": 0.279224, "token_totals": { "input_tokens": 533549683, "output_tokens": 52574268, "reasoning_tokens": 45760878, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 54.678, "latency": 1561.879, "stderr": 0.58, "cost_per_test": 1.201399, "token_totals": { "input_tokens": 365895891, "output_tokens": 27874175, "reasoning_tokens": null, "cache_read_tokens": 60248405, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 54.436, "latency": 1449.852, "stderr": 2.071, "cost_per_test": 3.629197, "token_totals": { "input_tokens": 651703808, "output_tokens": 67770045, "reasoning_tokens": 58991995, "cache_read_tokens": 334920434, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 54.138, "latency": 1849.0, "stderr": 2.132, "cost_per_test": 1.636603, "token_totals": { "input_tokens": 1125304863, "output_tokens": 207972605, "reasoning_tokens": 195320438, "cache_read_tokens": 434937131, "cache_write_tokens": 690320292 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 53.918, "latency": 547.397, "stderr": 0.159, "cost_per_test": 4.218538, "token_totals": { "input_tokens": 309993285, "output_tokens": 17301258, "reasoning_tokens": null, "cache_read_tokens": 82013466, "cache_write_tokens": 227923817 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 53.909, "latency": 792.086, "stderr": 0.518, "cost_per_test": 0.750759, "token_totals": { "input_tokens": 164610121, "output_tokens": 10102005, "reasoning_tokens": null, "cache_read_tokens": 75956629, "cache_write_tokens": 88646581 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 53.756, "latency": 1165.258, "stderr": 0.846, "cost_per_test": 1.253419, "token_totals": { "input_tokens": 322058918, "output_tokens": 6216567, "reasoning_tokens": 4178719, "cache_read_tokens": 276042752, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 53.682, "latency": 1000.937, "stderr": 0.67, "cost_per_test": 1.657732, "token_totals": { "input_tokens": 495041785, "output_tokens": 24551729, "reasoning_tokens": 16279727, "cache_read_tokens": 260943147, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 53.54, "latency": 692.672, "stderr": 2.078, "cost_per_test": 6.81746, "token_totals": { "input_tokens": 240568066, "output_tokens": 22397666, "reasoning_tokens": 16813955, "cache_read_tokens": 70591920, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 53.481, "latency": 364.376, "stderr": 0.39, "cost_per_test": 0.207465, "token_totals": { "input_tokens": 354713236, "output_tokens": 33772547, "reasoning_tokens": 28648435, "cache_read_tokens": 182250763, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 53.114, "latency": 285.753, "stderr": 0.374, "cost_per_test": 1.913224, "token_totals": { "input_tokens": 460080913, "output_tokens": 16314784, "reasoning_tokens": 10722175, "cache_read_tokens": 282968006, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.7": { "accuracy": 52.251, "latency": 932.724, "stderr": 0.355, "cost_per_test": 2.719136, "token_totals": { "input_tokens": 827415903, "output_tokens": 23428536, "reasoning_tokens": 15352966, "cache_read_tokens": 444856320, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 52.033, "latency": 925.46, "stderr": 0.308, "cost_per_test": 1.619399, "token_totals": { "input_tokens": 256899959, "output_tokens": 22298636, "reasoning_tokens": 16662434, "cache_read_tokens": 75246545, "cache_write_tokens": 181603278 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/ember-1": { "accuracy": 51.853, "latency": 798.612, "stderr": 2.124, "cost_per_test": 1.95445, "token_totals": { "input_tokens": 435525965, "output_tokens": 13497155, "reasoning_tokens": 8350801, "cache_read_tokens": 233160321, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 50.671, "latency": 747.585, "stderr": 0.324, "cost_per_test": 0.619283, "token_totals": { "input_tokens": 464025197, "output_tokens": 20770861, "reasoning_tokens": 0, "cache_read_tokens": 254565035, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 50.593, "latency": 1174.928, "stderr": 0.578, "cost_per_test": 1.236977, "token_totals": { "input_tokens": 313950822, "output_tokens": 28653788, "reasoning_tokens": 23202069, "cache_read_tokens": 139669632, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 50.393, "latency": 974.518, "stderr": 0.249, "cost_per_test": 0.880465, "token_totals": { "input_tokens": 419761736, "output_tokens": 30621918, "reasoning_tokens": 25089786, "cache_read_tokens": 218761173, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 49.873, "latency": 897.881, "stderr": 0.232, "cost_per_test": 0.124338, "token_totals": { "input_tokens": 413604271, "output_tokens": 45731435, "reasoning_tokens": 41794427, "cache_read_tokens": 199389488, "cache_write_tokens": 214114651 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 49.517, "latency": 490.245, "stderr": 0.465, "cost_per_test": 0.276154, "token_totals": { "input_tokens": 406112640, "output_tokens": 27423276, "reasoning_tokens": 22726163, "cache_read_tokens": 213584768, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-sol": { "accuracy": 49.05, "latency": 539.907, "stderr": 0.58, "cost_per_test": 2.122121, "token_totals": { "input_tokens": 416329027, "output_tokens": 22746494, "reasoning_tokens": 18774690, "cache_read_tokens": 155710451, "cache_write_tokens": 260528040 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 48.349, "latency": 289.014, "stderr": 0.458, "cost_per_test": 1.156495, "token_totals": { "input_tokens": 346429154, "output_tokens": 13008523, "reasoning_tokens": 7208719, "cache_read_tokens": 193601579, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 48.269, "latency": 496.894, "stderr": 0.437, "cost_per_test": 0.32345, "token_totals": { "input_tokens": 302569216, "output_tokens": 10069655, "reasoning_tokens": null, "cache_read_tokens": 126041678, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 46.597, "latency": 1082.125, "stderr": 0.925, "cost_per_test": 0.996264, "token_totals": { "input_tokens": 882963514, "output_tokens": 24911364, "reasoning_tokens": null, "cache_read_tokens": 647224491, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 45.36, "latency": 2153.503, "stderr": 0.447, "cost_per_test": 1.201815, "token_totals": { "input_tokens": 609985753, "output_tokens": 72941360, "reasoning_tokens": 70078573, "cache_read_tokens": 362827776, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 44.083, "latency": 1526.183, "stderr": 0.653, "cost_per_test": 0.879086, "token_totals": { "input_tokens": 508078472, "output_tokens": 22058372, "reasoning_tokens": 17233163, "cache_read_tokens": 284032811, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 42.982, "latency": 208.258, "stderr": 1.206, "cost_per_test": 1.591906, "token_totals": { "input_tokens": 365193664, "output_tokens": 7730374, "reasoning_tokens": 5787767, "cache_read_tokens": 123797501, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 41.502, "latency": 497.373, "stderr": 1.193, "cost_per_test": 0.206445, "token_totals": { "input_tokens": 507567154, "output_tokens": 8732144, "reasoning_tokens": 5054819, "cache_read_tokens": 314066453, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 41.257, "latency": 994.255, "stderr": 0.832, "cost_per_test": 0.230271, "token_totals": { "input_tokens": 696484546, "output_tokens": 28023249, "reasoning_tokens": null, "cache_read_tokens": 580562347, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-2.5": { "accuracy": 18.894, "latency": 92.482, "stderr": 0.216, "cost_per_test": 0.106441, "token_totals": { "input_tokens": 376083870, "output_tokens": 5831330, "reasoning_tokens": 4955938, "cache_read_tokens": 191920653, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "emb": { "anthropic/claude-fable-5-1": { "accuracy": 76.67, "latency": 1771.558, "stderr": 2.085, "cost_per_test": 15.93249, "token_totals": { "input_tokens": 454757375, "output_tokens": 12499169, "reasoning_tokens": 8150917, "cache_read_tokens": 378586242, "cache_write_tokens": 76164267 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 75.938, "latency": 2106.434, "stderr": 2.383, "cost_per_test": 11.018974, "token_totals": { "input_tokens": 922243024, "output_tokens": 23704546, "reasoning_tokens": 18569025, "cache_read_tokens": 861165257, "cache_write_tokens": 61067293 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 75.713, "latency": 2204.991, "stderr": 2.437, "cost_per_test": 8.792863, "token_totals": { "input_tokens": 2367951142, "output_tokens": 29328538, "reasoning_tokens": 22320203, "cache_read_tokens": 2311427347, "cache_write_tokens": 56507215 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 75.244, "latency": 2305.821, "stderr": 2.459, "cost_per_test": 11.8029, "token_totals": { "input_tokens": 1127460088, "output_tokens": 14257556, "reasoning_tokens": 9497334, "cache_read_tokens": 945025185, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 73.666, "latency": 1594.153, "stderr": 2.468, "cost_per_test": 12.354419, "token_totals": { "input_tokens": 345067688, "output_tokens": 11953058, "reasoning_tokens": null, "cache_read_tokens": 318538087, "cache_write_tokens": 26523742 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 73.556, "latency": 1558.673, "stderr": 2.24, "cost_per_test": 6.071566, "token_totals": { "input_tokens": 686017503, "output_tokens": 12323715, "reasoning_tokens": 7957163, "cache_read_tokens": 656949908, "cache_write_tokens": 29058427 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 72.339, "latency": 590.258, "stderr": 2.373, "cost_per_test": 6.010773, "token_totals": { "input_tokens": 318866006, "output_tokens": 5457571, "reasoning_tokens": 3316634, "cache_read_tokens": 276762112, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 72.196, "latency": 769.197, "stderr": 2.42, "cost_per_test": 8.243862, "token_totals": { "input_tokens": 2960728904, "output_tokens": 21722815, "reasoning_tokens": 13162823, "cache_read_tokens": 2787511478, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 71.7, "latency": 804.28, "stderr": 2.475, "cost_per_test": 5.827063, "token_totals": { "input_tokens": 123989946, "output_tokens": 5542631, "reasoning_tokens": 3051513, "cache_read_tokens": 104449111, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 71.53, "latency": 621.736, "stderr": 2.351, "cost_per_test": 1.276882, "token_totals": { "input_tokens": 216104819, "output_tokens": 5241605, "reasoning_tokens": 3222968, "cache_read_tokens": 204982294, "cache_write_tokens": 11114719 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 71.326, "latency": 650.914, "stderr": 2.262, "cost_per_test": 5.694001, "token_totals": { "input_tokens": 1686304951, "output_tokens": 18606388, "reasoning_tokens": 7782800, "cache_read_tokens": 1546827567, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 70.773, "latency": 1107.811, "stderr": 2.712, "cost_per_test": 1.143199, "token_totals": { "input_tokens": 129631093, "output_tokens": 6056928, "reasoning_tokens": 3495567, "cache_read_tokens": 112922114, "cache_write_tokens": 16703171 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 69.373, "latency": 2632.285, "stderr": 2.606, "cost_per_test": 12.05844, "token_totals": { "input_tokens": 833619980, "output_tokens": 22136588, "reasoning_tokens": null, "cache_read_tokens": 790543547, "cache_write_tokens": 43067000 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 68.515, "latency": 1041.052, "stderr": 2.809, "cost_per_test": 0.141708, "token_totals": { "input_tokens": 362285754, "output_tokens": 11007243, "reasoning_tokens": 8590626, "cache_read_tokens": 332678280, "cache_write_tokens": 29598418 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 67.426, "latency": 829.829, "stderr": 3.056, "cost_per_test": 2.546868, "token_totals": { "input_tokens": 1115802604, "output_tokens": 13342216, "reasoning_tokens": 8360823, "cache_read_tokens": 1083342803, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 67.116, "latency": 660.047, "stderr": 2.936, "cost_per_test": 0.366502, "token_totals": { "input_tokens": 653086981, "output_tokens": 6955414, "reasoning_tokens": 4412805, "cache_read_tokens": 604381577, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 66.991, "latency": 2097.023, "stderr": 3.045, "cost_per_test": 6.481284, "token_totals": { "input_tokens": 531961928, "output_tokens": 15529729, "reasoning_tokens": 10549516, "cache_read_tokens": 448911360, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 66.677, "latency": 741.42, "stderr": 2.873, "cost_per_test": 3.074655, "token_totals": { "input_tokens": 410193956, "output_tokens": 9698383, "reasoning_tokens": 7045672, "cache_read_tokens": 393497338, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 66.316, "latency": 2985.704, "stderr": 3.006, "cost_per_test": 10.293794, "token_totals": { "input_tokens": 3099396585, "output_tokens": 25999381, "reasoning_tokens": null, "cache_read_tokens": 3034388542, "cache_write_tokens": 64983558 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 66.205, "latency": 813.024, "stderr": 3.11, "cost_per_test": 2.193037, "token_totals": { "input_tokens": 314065634, "output_tokens": 7024530, "reasoning_tokens": 4854711, "cache_read_tokens": 286789081, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 65.459, "latency": 1521.906, "stderr": 2.709, "cost_per_test": 0.108634, "token_totals": { "input_tokens": 891374421, "output_tokens": 16056047, "reasoning_tokens": 12655956, "cache_read_tokens": 861570496, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 63.548, "latency": 1123.555, "stderr": 2.752, "cost_per_test": 5.173115, "token_totals": { "input_tokens": 1617865425, "output_tokens": 16984726, "reasoning_tokens": 10956816, "cache_read_tokens": 1520002205, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/ember-1": { "accuracy": 63.368, "latency": 1867.313, "stderr": 2.949, "cost_per_test": 2.459539, "token_totals": { "input_tokens": 287599624, "output_tokens": 7246798, "reasoning_tokens": 4945223, "cache_read_tokens": 266899209, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 62.858, "latency": 3313.924, "stderr": 3.141, "cost_per_test": 0.328999, "token_totals": { "input_tokens": 847310790, "output_tokens": 17554862, "reasoning_tokens": 14594001, "cache_read_tokens": 813521152, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.6": { "accuracy": 62.727, "latency": 3178.533, "stderr": 3.085, "cost_per_test": 3.056382, "token_totals": { "input_tokens": 292867165, "output_tokens": 18238122, "reasoning_tokens": 12738109, "cache_read_tokens": 269312000, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 62.712, "latency": 1374.999, "stderr": 2.956, "cost_per_test": 3.552349, "token_totals": { "input_tokens": 1152349029, "output_tokens": 12397789, "reasoning_tokens": 7178696, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 60.066, "latency": 3467.209, "stderr": 3.352, "cost_per_test": 2.655924, "token_totals": { "input_tokens": 599112401, "output_tokens": 12051178, "reasoning_tokens": 8874470, "cache_read_tokens": 572733440, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 59.015, "latency": 1082.902, "stderr": 4.17, "cost_per_test": 1.146892, "token_totals": { "input_tokens": 1240305362, "output_tokens": 21428591, "reasoning_tokens": 16654436, "cache_read_tokens": 1207542704, "cache_write_tokens": 32752108 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 58.076, "latency": 1839.41, "stderr": 3.136, "cost_per_test": 0.787396, "token_totals": { "input_tokens": 824410363, "output_tokens": 10431429, "reasoning_tokens": 7444891, "cache_read_tokens": 799662720, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 57.208, "latency": 780.458, "stderr": 3.227, "cost_per_test": 0.234152, "token_totals": { "input_tokens": 893420650, "output_tokens": 13279281, "reasoning_tokens": 10184478, "cache_read_tokens": 884618493, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "stepfun/step-5-preview": { "accuracy": 57.116, "latency": 2615.096, "stderr": 3.165, "cost_per_test": 1.329318, "token_totals": { "input_tokens": 1262287639, "output_tokens": 14703425, "reasoning_tokens": 0, "cache_read_tokens": 1227785728, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 56.978, "latency": 1027.476, "stderr": 3.127, "cost_per_test": 2.313636, "token_totals": { "input_tokens": 829404547, "output_tokens": 10741088, "reasoning_tokens": 5180580, "cache_read_tokens": 769467634, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 56.977, "latency": 1574.097, "stderr": 3.391, "cost_per_test": 0.383709, "token_totals": { "input_tokens": 1018385162, "output_tokens": 15135599, "reasoning_tokens": 11153857, "cache_read_tokens": 1008679680, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 56.344, "latency": 2172.231, "stderr": 3.319, "cost_per_test": 3.787827, "token_totals": { "input_tokens": 942177449, "output_tokens": 18177772, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 56.213, "latency": 4904.981, "stderr": 3.22, "cost_per_test": 4.839324, "token_totals": { "input_tokens": 766628987, "output_tokens": 17347923, "reasoning_tokens": null, "cache_read_tokens": 509441536, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 55.228, "latency": 1747.98, "stderr": 3.112, "cost_per_test": 0.221264, "token_totals": { "input_tokens": 455429765, "output_tokens": 11928848, "reasoning_tokens": 6975136, "cache_read_tokens": 430971584, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.5": { "accuracy": 52.911, "latency": 1107.838, "stderr": 3.111, "cost_per_test": 1.55492, "token_totals": { "input_tokens": 150960387, "output_tokens": 7436273, "reasoning_tokens": 2117149, "cache_read_tokens": 129340672, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 52.803, "latency": 2247.236, "stderr": 3.059, "cost_per_test": 1.08141, "token_totals": { "input_tokens": 864870937, "output_tokens": 15254147, "reasoning_tokens": 11469965, "cache_read_tokens": 855589504, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 52.616, "latency": 708.582, "stderr": 2.974, "cost_per_test": 3.769246, "token_totals": { "input_tokens": 703037870, "output_tokens": 5442617, "reasoning_tokens": 2522069, "cache_read_tokens": 656983985, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 51.623, "latency": 822.144, "stderr": 3.053, "cost_per_test": 0.796757, "token_totals": { "input_tokens": 541981728, "output_tokens": 9803133, "reasoning_tokens": 3184172, "cache_read_tokens": 527403648, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 47.759, "latency": 1890.343, "stderr": 2.946, "cost_per_test": 2.091762, "token_totals": { "input_tokens": 1398807070, "output_tokens": 13708644, "reasoning_tokens": null, "cache_read_tokens": 1372552733, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 45.426, "latency": 2601.215, "stderr": 3.645, "cost_per_test": 2.146984, "token_totals": { "input_tokens": 688505179, "output_tokens": 26360621, "reasoning_tokens": 22515833, "cache_read_tokens": 616310016, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 38.362, "latency": 1581.835, "stderr": 3.273, "cost_per_test": 1.671783, "token_totals": { "input_tokens": 658436029, "output_tokens": 9281665, "reasoning_tokens": null, "cache_read_tokens": 641195264, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 31.762, "latency": 1173.625, "stderr": 2.956, "cost_per_test": 0.249301, "token_totals": { "input_tokens": 254752442, "output_tokens": 6362193, "reasoning_tokens": null, "cache_read_tokens": 245337472, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-2.5": { "accuracy": 8.789, "latency": 300.757, "stderr": 1.275, "cost_per_test": 0.417608, "token_totals": { "input_tokens": 407236592, "output_tokens": 5531340, "reasoning_tokens": 5399594, "cache_read_tokens": 238888010, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "code_migration": { "anthropic/claude-sonnet-5-5": { "accuracy": 70.131, "latency": 12151.11, "stderr": 4.777, "cost_per_test": 66.963572, "token_totals": { "input_tokens": 18672812024, "output_tokens": 47538207, "reasoning_tokens": null, "cache_read_tokens": 18443798779, "cache_write_tokens": 228955399 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 69.503, "latency": 8091.746, "stderr": 4.856, "cost_per_test": 63.053568, "token_totals": { "input_tokens": 8089377797, "output_tokens": 22408176, "reasoning_tokens": 5388215, "cache_read_tokens": 7545427916, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 68.901, "latency": 10312.167, "stderr": 4.723, "cost_per_test": 104.165309, "token_totals": { "input_tokens": 14447217682, "output_tokens": 38623427, "reasoning_tokens": null, "cache_read_tokens": 14262241543, "cache_write_tokens": 184928335 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 68.462, "latency": 3477.726, "stderr": 4.581, "cost_per_test": 44.392054, "token_totals": { "input_tokens": 1029187893, "output_tokens": 9499581, "reasoning_tokens": 3212601, "cache_read_tokens": 977435512, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 63.309, "latency": 5616.826, "stderr": 5.038, "cost_per_test": 6.531761, "token_totals": { "input_tokens": 1064117159, "output_tokens": 12923576, "reasoning_tokens": 5911286, "cache_read_tokens": 1038392459, "cache_write_tokens": 25621006 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 60.033, "latency": 15033.475, "stderr": 5.359, "cost_per_test": 72.424868, "token_totals": { "input_tokens": 10264913505, "output_tokens": 29788772, "reasoning_tokens": null, "cache_read_tokens": 8854118675, "cache_write_tokens": 1410755640 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 56.92, "latency": 5089.673, "stderr": 4.599, "cost_per_test": 14.564951, "token_totals": { "input_tokens": 2322801542, "output_tokens": 10239389, "reasoning_tokens": 3831388, "cache_read_tokens": 2297501469, "cache_write_tokens": 25272650 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 56.68, "latency": 10031.127, "stderr": 5.055, "cost_per_test": 55.28024, "token_totals": { "input_tokens": 5714388105, "output_tokens": 20649057, "reasoning_tokens": null, "cache_read_tokens": 5642498395, "cache_write_tokens": 71858364 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 54.635, "latency": 6526.389, "stderr": 5.222, "cost_per_test": 108.499119, "token_totals": { "input_tokens": 5597593771, "output_tokens": 20933498, "reasoning_tokens": null, "cache_read_tokens": 5561285040, "cache_write_tokens": 36280807 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 52.397, "latency": 3444.398, "stderr": 4.956, "cost_per_test": 25.76956, "token_totals": { "input_tokens": 1727817656, "output_tokens": 7915451, "reasoning_tokens": 2058586, "cache_read_tokens": 1641043968, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 48.344, "latency": 4682.008, "stderr": 4.769, "cost_per_test": 14.559091, "token_totals": { "input_tokens": 5439866954, "output_tokens": 17951662, "reasoning_tokens": 7222274, "cache_read_tokens": 5385918220, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 48.248, "latency": 2978.471, "stderr": 4.816, "cost_per_test": 10.959637, "token_totals": { "input_tokens": 1603103372, "output_tokens": 7887791, "reasoning_tokens": 2595938, "cache_read_tokens": 1542349503, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 46.942, "latency": 4454.295, "stderr": 4.833, "cost_per_test": 30.057323, "token_totals": { "input_tokens": 2581769862, "output_tokens": 18356907, "reasoning_tokens": null, "cache_read_tokens": 2555676483, "cache_write_tokens": 26072355 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 46.854, "latency": 6779.828, "stderr": 4.852, "cost_per_test": 0.753694, "token_totals": { "input_tokens": 4064264660, "output_tokens": 15996575, "reasoning_tokens": 9301483, "cache_read_tokens": 4049037820, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 46.812, "latency": 7628.207, "stderr": 4.75, "cost_per_test": 3.436542, "token_totals": { "input_tokens": 3662384405, "output_tokens": 15049371, "reasoning_tokens": 8167976, "cache_read_tokens": 3623399040, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Openrouter", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 45.921, "latency": 4625.77, "stderr": 4.787, "cost_per_test": 5.835634, "token_totals": { "input_tokens": 5484573973, "output_tokens": 29269470, "reasoning_tokens": null, "cache_read_tokens": 5434154715, "cache_write_tokens": 50391718 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 44.774, "latency": 3404.333, "stderr": 4.762, "cost_per_test": 2.797004, "token_totals": { "input_tokens": 3661685805, "output_tokens": 9104000, "reasoning_tokens": 3615927, "cache_read_tokens": 3481296159, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 44.603, "latency": 5425.271, "stderr": 4.957, "cost_per_test": 14.765608, "token_totals": { "input_tokens": 1617793063, "output_tokens": 14398758, "reasoning_tokens": 7964660, "cache_read_tokens": 1574035200, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 44.377, "latency": 2950.117, "stderr": 5.004, "cost_per_test": 0.547293, "token_totals": { "input_tokens": 1720559222, "output_tokens": 11387800, "reasoning_tokens": 6330094, "cache_read_tokens": 1701613975, "cache_write_tokens": 18922708 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 44.089, "latency": 6929.224, "stderr": 4.98, "cost_per_test": 33.545739, "token_totals": { "input_tokens": 9033717591, "output_tokens": 25196607, "reasoning_tokens": null, "cache_read_tokens": 8982964678, "cache_write_tokens": 50709475 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 43.198, "latency": 14498.978, "stderr": 4.813, "cost_per_test": 22.959059, "token_totals": { "input_tokens": 4596186809, "output_tokens": 16492838, "reasoning_tokens": null, "cache_read_tokens": 4417489768, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.7": { "accuracy": 43.159, "latency": 4057.679, "stderr": 4.598, "cost_per_test": 35.977302, "token_totals": { "input_tokens": 2200666548, "output_tokens": 10780365, "reasoning_tokens": 4987474, "cache_read_tokens": 2075628160, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 42.329, "latency": 7395.352, "stderr": 4.792, "cost_per_test": 3.957576, "token_totals": { "input_tokens": 7771918239, "output_tokens": 19223061, "reasoning_tokens": 9157654, "cache_read_tokens": 7476027520, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 42.31, "latency": 10065.11, "stderr": 4.85, "cost_per_test": 0.682037, "token_totals": { "input_tokens": 3362298201, "output_tokens": 19744960, "reasoning_tokens": 12471457, "cache_read_tokens": 3329895936, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 42.192, "latency": 4061.108, "stderr": 4.561, "cost_per_test": 9.853488, "token_totals": { "input_tokens": 2834033709, "output_tokens": 12223208, "reasoning_tokens": 4663206, "cache_read_tokens": 2691783871, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 41.545, "latency": 12582.223, "stderr": 4.905, "cost_per_test": 19.204104, "token_totals": { "input_tokens": 9922831954, "output_tokens": 21697307, "reasoning_tokens": 9297600, "cache_read_tokens": 9342697216, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 40.368, "latency": 11194.48, "stderr": 4.998, "cost_per_test": 0.45437, "token_totals": { "input_tokens": 5518086732, "output_tokens": 25383280, "reasoning_tokens": 14174151, "cache_read_tokens": 5470014528, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "stepfun/step-5-preview": { "accuracy": 39.459, "latency": 12648.433, "stderr": 5.118, "cost_per_test": 7.697813, "token_totals": { "input_tokens": 7897085309, "output_tokens": 21050698, "reasoning_tokens": 0, "cache_read_tokens": 7852930048, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "grok/grok-4.5": { "accuracy": 38.405, "latency": 1444.03, "stderr": 4.717, "cost_per_test": 3.485568, "token_totals": { "input_tokens": 434596005, "output_tokens": 6923870, "reasoning_tokens": 1464472, "cache_read_tokens": 419658752, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 31.558, "latency": 14719.553, "stderr": 4.478, "cost_per_test": 13.802526, "token_totals": { "input_tokens": 3345153832, "output_tokens": 10663834, "reasoning_tokens": 5329305, "cache_read_tokens": 3120951990, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/ember-1": { "accuracy": 30.46, "latency": 14933.492, "stderr": 5.142, "cost_per_test": 34.811848, "token_totals": { "input_tokens": 6034561494, "output_tokens": 17742042, "reasoning_tokens": 6682531, "cache_read_tokens": 5966780237, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 29.403, "latency": 1895.227, "stderr": 4.378, "cost_per_test": 3.925026, "token_totals": { "input_tokens": 1195020481, "output_tokens": 8130941, "reasoning_tokens": 3618470, "cache_read_tokens": 1165941081, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 28.997, "latency": 14661.487, "stderr": 4.688, "cost_per_test": 12.847904, "token_totals": { "input_tokens": 3179208698, "output_tokens": 13304977, "reasoning_tokens": 4012904, "cache_read_tokens": 2988589895, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 27.148, "latency": 1162.652, "stderr": 4.407, "cost_per_test": 5.896207, "token_totals": { "input_tokens": 1268932455, "output_tokens": 8297778, "reasoning_tokens": 4261992, "cache_read_tokens": 1193161490, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 26.212, "latency": 1719.889, "stderr": 4.335, "cost_per_test": 1.04578, "token_totals": { "input_tokens": 770033456, "output_tokens": 5867778, "reasoning_tokens": 1599375, "cache_read_tokens": 765168768, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 23.163, "latency": 15822.169, "stderr": 4.493, "cost_per_test": 10.545353, "token_totals": { "input_tokens": 2163200307, "output_tokens": 12849749, "reasoning_tokens": 6884264, "cache_read_tokens": 2139278080, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 22.799, "latency": 17496.668, "stderr": 4.219, "cost_per_test": 44.587214, "token_totals": { "input_tokens": 3281487252, "output_tokens": 18332480, "reasoning_tokens": null, "cache_read_tokens": 1441923072, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 20.879, "latency": 4154.455, "stderr": 4.24, "cost_per_test": 6.593315, "token_totals": { "input_tokens": 3094038387, "output_tokens": 8632886, "reasoning_tokens": null, "cache_read_tokens": 3053969418, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 20.402, "latency": 2976.567, "stderr": 4.365, "cost_per_test": 0.237176, "token_totals": { "input_tokens": 1078559679, "output_tokens": 6929600, "reasoning_tokens": 2495015, "cache_read_tokens": 1067160640, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 17.911, "latency": 703.322, "stderr": 4.278, "cost_per_test": 1.592808, "token_totals": { "input_tokens": 190208614, "output_tokens": 2033653, "reasoning_tokens": 949508, "cache_read_tokens": 172927611, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 16.042, "latency": 15445.227, "stderr": 4.134, "cost_per_test": 13.951333, "token_totals": { "input_tokens": 2053108137, "output_tokens": 11346877, "reasoning_tokens": 5449513, "cache_read_tokens": 2030185983, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 13.32, "latency": 1241.908, "stderr": 3.885, "cost_per_test": 0.612969, "token_totals": { "input_tokens": 507102342, "output_tokens": 3760880, "reasoning_tokens": null, "cache_read_tokens": 500832128, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 13.015, "latency": 1774.029, "stderr": 3.851, "cost_per_test": 1.967707, "token_totals": { "input_tokens": 825775721, "output_tokens": 8502165, "reasoning_tokens": 5791384, "cache_read_tokens": 788337152, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 11.759, "latency": 1722.389, "stderr": 3.575, "cost_per_test": 2.830671, "token_totals": { "input_tokens": 803892329, "output_tokens": 3303883, "reasoning_tokens": null, "cache_read_tokens": 794874880, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-2.5": { "accuracy": 4.863, "latency": 1044.108, "stderr": 1.801, "cost_per_test": 0.738732, "token_totals": { "input_tokens": 427771621, "output_tokens": 2921137, "reasoning_tokens": 2787023, "cache_read_tokens": 248786079, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "legal_research": { "meta/muse_spark_1_3_max": { "accuracy": 55.288, "latency": 317.461, "stderr": 3.456, "cost_per_test": 0.588812, "token_totals": { "input_tokens": 240995118, "output_tokens": 10646285, "reasoning_tokens": 6279342, "cache_read_tokens": 205258324, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 55.288, "latency": 1860.719, "stderr": 3.456, "cost_per_test": 6.762498, "token_totals": { "input_tokens": 626960894, "output_tokens": 23871466, "reasoning_tokens": 9873124, "cache_read_tokens": 540621000, "cache_write_tokens": 86242293 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 55.288, "latency": 3449.382, "stderr": 3.456, "cost_per_test": 23.055789, "token_totals": { "input_tokens": 1085827683, "output_tokens": 50734245, "reasoning_tokens": 31513293, "cache_read_tokens": 927334456, "cache_write_tokens": 158405763 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 54.808, "latency": 1881.045, "stderr": 3.459, "cost_per_test": 6.44908, "token_totals": { "input_tokens": 915877177, "output_tokens": 24432059, "reasoning_tokens": 11886477, "cache_read_tokens": 739668768, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 50.481, "latency": 6749.531, "stderr": 3.475, "cost_per_test": 25.163725, "token_totals": { "input_tokens": 1711216726, "output_tokens": 144935934, "reasoning_tokens": 126915044, "cache_read_tokens": 1456137656, "cache_write_tokens": 254987053 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 49.519, "latency": 1337.604, "stderr": 3.475, "cost_per_test": 9.791426, "token_totals": { "input_tokens": 210052964, "output_tokens": 22537892, "reasoning_tokens": 0, "cache_read_tokens": 150055095, "cache_write_tokens": 59958741 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 49.038, "latency": 3110.171, "stderr": 3.475, "cost_per_test": 2.241388, "token_totals": { "input_tokens": 458279584, "output_tokens": 55494507, "reasoning_tokens": 0, "cache_read_tokens": 368033787, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.6": { "accuracy": 48.077, "latency": 1504.077, "stderr": 3.473, "cost_per_test": 1.530204, "token_totals": { "input_tokens": 274776010, "output_tokens": 12377024, "reasoning_tokens": 7368696, "cache_read_tokens": 203687808, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 48.077, "latency": 4631.383, "stderr": 3.473, "cost_per_test": 21.605566, "token_totals": { "input_tokens": 1487117357, "output_tokens": 70127005, "reasoning_tokens": 61207392, "cache_read_tokens": 1241425920, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 48.077, "latency": 5588.989, "stderr": 3.473, "cost_per_test": 14.826571, "token_totals": { "input_tokens": 2650038387, "output_tokens": 173348390, "reasoning_tokens": 151205430, "cache_read_tokens": 2338418360, "cache_write_tokens": 311466299 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 47.596, "latency": 3677.636, "stderr": 3.471, "cost_per_test": 2.486351, "token_totals": { "input_tokens": 544286180, "output_tokens": 42522887, "reasoning_tokens": 35664649, "cache_read_tokens": 472313472, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.7": { "accuracy": 47.115, "latency": 1274.337, "stderr": 3.469, "cost_per_test": 4.920812, "token_totals": { "input_tokens": 476673717, "output_tokens": 14028919, "reasoning_tokens": 9121242, "cache_read_tokens": 350504320, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 47.115, "latency": 1816.322, "stderr": 3.469, "cost_per_test": 0.181201, "token_totals": { "input_tokens": 247379616, "output_tokens": 20743987, "reasoning_tokens": 15408757, "cache_read_tokens": 204771712, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 46.154, "latency": 760.122, "stderr": 3.465, "cost_per_test": 3.469601, "token_totals": { "input_tokens": 672994446, "output_tokens": 6248749, "reasoning_tokens": 3873135, "cache_read_tokens": 651592955, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 45.192, "latency": 3839.511, "stderr": 3.459, "cost_per_test": 0.846372, "token_totals": { "input_tokens": 312749338, "output_tokens": 50001075, "reasoning_tokens": 43836379, "cache_read_tokens": 264949824, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "fireworks/ember-1": { "accuracy": 44.712, "latency": 2313.552, "stderr": 3.456, "cost_per_test": 3.674091, "token_totals": { "input_tokens": 899320434, "output_tokens": 18199298, "reasoning_tokens": 11913573, "cache_read_tokens": 817311062, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 43.75, "latency": 306.893, "stderr": 3.448, "cost_per_test": 0.508097, "token_totals": { "input_tokens": 163340498, "output_tokens": 5837394, "reasoning_tokens": 2676881, "cache_read_tokens": 113477050, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 43.75, "latency": 714.036, "stderr": 3.448, "cost_per_test": 2.820942, "token_totals": { "input_tokens": 136489421, "output_tokens": 12617848, "reasoning_tokens": 0, "cache_read_tokens": 98212592, "cache_write_tokens": 38178928 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 43.269, "latency": 4911.734, "stderr": 3.444, "cost_per_test": 1.633459, "token_totals": { "input_tokens": 1645146593, "output_tokens": 246336648, "reasoning_tokens": 230649495, "cache_read_tokens": 1396606112, "cache_write_tokens": 248419597 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 41.827, "latency": 1545.997, "stderr": 3.428, "cost_per_test": 2.719446, "token_totals": { "input_tokens": 435277342, "output_tokens": 29025915, "reasoning_tokens": 0, "cache_read_tokens": 353355756, "cache_write_tokens": 81742641 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 41.346, "latency": 701.783, "stderr": 3.423, "cost_per_test": 0.24793, "token_totals": { "input_tokens": 410443550, "output_tokens": 29229774, "reasoning_tokens": 25253265, "cache_read_tokens": 362718592, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 41.346, "latency": 4172.869, "stderr": 3.423, "cost_per_test": 7.907299, "token_totals": { "input_tokens": 1570540015, "output_tokens": 81682466, "reasoning_tokens": 73169720, "cache_read_tokens": 1375861858, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 40.865, "latency": 530.813, "stderr": 3.417, "cost_per_test": 0.650063, "token_totals": { "input_tokens": 193686912, "output_tokens": 8979986, "reasoning_tokens": 5305335, "cache_read_tokens": 133646110, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 40.865, "latency": 2527.177, "stderr": 3.417, "cost_per_test": 1.088017, "token_totals": { "input_tokens": 433434003, "output_tokens": 35408474, "reasoning_tokens": 30213976, "cache_read_tokens": 380911360, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "stepfun/step-5-preview": { "accuracy": 40.385, "latency": 2168.413, "stderr": 3.41, "cost_per_test": 0.838645, "token_totals": { "input_tokens": 683339325, "output_tokens": 23833353, "reasoning_tokens": 0, "cache_read_tokens": 604305152, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-astra": { "accuracy": 39.423, "latency": 1491.339, "stderr": 3.397, "cost_per_test": 10.577143, "token_totals": { "input_tokens": 290661092, "output_tokens": 21448775, "reasoning_tokens": 16238030, "cache_read_tokens": 215071044, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 38.942, "latency": 324.318, "stderr": 3.389, "cost_per_test": 1.630498, "token_totals": { "input_tokens": 581263116, "output_tokens": 14420656, "reasoning_tokens": 10782340, "cache_read_tokens": 474745174, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 38.462, "latency": 2060.48, "stderr": 3.381, "cost_per_test": 3.384561, "token_totals": { "input_tokens": 329856713, "output_tokens": 20763888, "reasoning_tokens": 15434407, "cache_read_tokens": 260063643, "cache_write_tokens": 68892844 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 37.981, "latency": 919.689, "stderr": 3.373, "cost_per_test": 0.070424, "token_totals": { "input_tokens": 371466896, "output_tokens": 20269368, "reasoning_tokens": 10496896, "cache_read_tokens": 310639232, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.5": { "accuracy": 37.981, "latency": 953.767, "stderr": 3.373, "cost_per_test": 1.086814, "token_totals": { "input_tokens": 128982108, "output_tokens": 6582496, "reasoning_tokens": 3272884, "cache_read_tokens": 74664576, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 36.538, "latency": 2373.859, "stderr": 3.347, "cost_per_test": 0.845008, "token_totals": { "input_tokens": 1755111665, "output_tokens": 88330527, "reasoning_tokens": 78699683, "cache_read_tokens": 1562540626, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 34.615, "latency": 288.96, "stderr": 3.307, "cost_per_test": 0.760397, "token_totals": { "input_tokens": 216485075, "output_tokens": 6173336, "reasoning_tokens": 3838872, "cache_read_tokens": 157677752, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 31.731, "latency": 5571.099, "stderr": 3.235, "cost_per_test": 5.32689, "token_totals": { "input_tokens": 1494014814, "output_tokens": 37508841, "reasoning_tokens": 0, "cache_read_tokens": 885782016, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 30.769, "latency": 277.753, "stderr": 3.208, "cost_per_test": 1.305281, "token_totals": { "input_tokens": 556323486, "output_tokens": 7365405, "reasoning_tokens": 5568107, "cache_read_tokens": 472254997, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 30.288, "latency": 1266.744, "stderr": 3.194, "cost_per_test": 0.275172, "token_totals": { "input_tokens": 310561183, "output_tokens": 24287114, "reasoning_tokens": 20969113, "cache_read_tokens": 261666816, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 30.288, "latency": 2547.747, "stderr": 3.194, "cost_per_test": 0.440296, "token_totals": { "input_tokens": 1145599233, "output_tokens": 60206390, "reasoning_tokens": 55010848, "cache_read_tokens": 1044290820, "cache_write_tokens": 100299736 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 29.808, "latency": 813.838, "stderr": 3.179, "cost_per_test": 0.342955, "token_totals": { "input_tokens": 273575187, "output_tokens": 6440429, "reasoning_tokens": 0, "cache_read_tokens": 219245518, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-6-sol": { "accuracy": 28.846, "latency": 1449.451, "stderr": 3.149, "cost_per_test": 4.897717, "token_totals": { "input_tokens": 709313788, "output_tokens": 26332564, "reasoning_tokens": 22035035, "cache_read_tokens": 634566483, "cache_write_tokens": 73881332 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 28.365, "latency": 869.377, "stderr": 3.133, "cost_per_test": 0.494044, "token_totals": { "input_tokens": 271552348, "output_tokens": 8856526, "reasoning_tokens": 0, "cache_read_tokens": 246578560, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 25.481, "latency": 774.823, "stderr": 3.029, "cost_per_test": 0.141548, "token_totals": { "input_tokens": 209190661, "output_tokens": 10262319, "reasoning_tokens": 0, "cache_read_tokens": 190714496, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 23.077, "latency": 1024.619, "stderr": 2.928, "cost_per_test": 0.682218, "token_totals": { "input_tokens": 341401669, "output_tokens": 11003300, "reasoning_tokens": 8033786, "cache_read_tokens": 300570368, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 20.673, "latency": 242.05, "stderr": 2.815, "cost_per_test": 1.100145, "token_totals": { "input_tokens": 354575717, "output_tokens": 2983139, "reasoning_tokens": 2206660, "cache_read_tokens": 321431187, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 15.865, "latency": 416.437, "stderr": 2.539, "cost_per_test": 0.082648, "token_totals": { "input_tokens": 179797926, "output_tokens": 3117976, "reasoning_tokens": 0, "cache_read_tokens": 147737472, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 12.5, "latency": 2256.339, "stderr": 2.299, "cost_per_test": 1.864673, "token_totals": { "input_tokens": 742970115, "output_tokens": 55568745, "reasoning_tokens": 53294787, "cache_read_tokens": 622977536, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 4.327, "latency": 47.609, "stderr": 1.414, "cost_per_test": 0.058112, "token_totals": { "input_tokens": 77109229, "output_tokens": 1908210, "reasoning_tokens": 1539184, "cache_read_tokens": 25000429, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "legal_agent_benchmark": { "meta/muse_spark_1_2": { "accuracy": 25.417, "latency": 1456.298, "stderr": 3.667, "cost_per_test": 2.08555, "token_totals": { "input_tokens": 608754402, "output_tokens": 10507621, "reasoning_tokens": 826686, "cache_read_tokens": 512433248, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 23.75, "latency": 882.844, "stderr": 3.547, "cost_per_test": 2.25729, "token_totals": { "input_tokens": 1051561571, "output_tokens": 12477171, "reasoning_tokens": 2210430, "cache_read_tokens": 1009226293, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 22.917, "latency": 1024.261, "stderr": 3.416, "cost_per_test": 2.75654, "token_totals": { "input_tokens": 835346220, "output_tokens": 8208587, "reasoning_tokens": 1656318, "cache_read_tokens": 690282338, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 19.583, "latency": 2634.915, "stderr": 3.31, "cost_per_test": 10.54836, "token_totals": { "input_tokens": 826582568, "output_tokens": 18828918, "reasoning_tokens": 5044847, "cache_read_tokens": 636080391, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 15.833, "latency": 2364.651, "stderr": 2.958, "cost_per_test": 4.181392, "token_totals": { "input_tokens": 826343789, "output_tokens": 12798926, "reasoning_tokens": null, "cache_read_tokens": 553737728, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.6": { "accuracy": 15.833, "latency": 2705.048, "stderr": 2.645, "cost_per_test": 4.008154, "token_totals": { "input_tokens": 655024488, "output_tokens": 16284504, "reasoning_tokens": 3704862, "cache_read_tokens": 617851648, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.5": { "accuracy": 12.917, "latency": 614.705, "stderr": 2.534, "cost_per_test": 2.01955, "token_totals": { "input_tokens": 220580311, "output_tokens": 9143344, "reasoning_tokens": 326470, "cache_read_tokens": 198038400, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 12.917, "latency": 837.646, "stderr": 2.679, "cost_per_test": 3.829773, "token_totals": { "input_tokens": 797226748, "output_tokens": 9686458, "reasoning_tokens": 3624789, "cache_read_tokens": 769409022, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.7": { "accuracy": 12.5, "latency": 2525.239, "stderr": 2.663, "cost_per_test": 11.130983, "token_totals": { "input_tokens": 1000748161, "output_tokens": 20963877, "reasoning_tokens": 8944068, "cache_read_tokens": 911133568, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 11.25, "latency": 1031.581, "stderr": 2.543, "cost_per_test": 0.085646, "token_totals": { "input_tokens": 751303984, "output_tokens": 11580281, "reasoning_tokens": 5709544, "cache_read_tokens": 715360896, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 11.25, "latency": 1613.035, "stderr": 2.148, "cost_per_test": 19.225253, "token_totals": { "input_tokens": 939267211, "output_tokens": 13345157, "reasoning_tokens": null, "cache_read_tokens": 880023896, "cache_write_tokens": 59236069 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 10.833, "latency": 1553.048, "stderr": 2.453, "cost_per_test": 0.216614, "token_totals": { "input_tokens": 632411237, "output_tokens": 11283302, "reasoning_tokens": 5261809, "cache_read_tokens": 600189312, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 10.417, "latency": 2952.433, "stderr": 2.287, "cost_per_test": 2.365468, "token_totals": { "input_tokens": 702241099, "output_tokens": 9014364, "reasoning_tokens": 3675937, "cache_read_tokens": 671264128, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 10.0, "latency": 1760.871, "stderr": 2.415, "cost_per_test": 3.656207, "token_totals": { "input_tokens": 1171742866, "output_tokens": 16120993, "reasoning_tokens": 5514918, "cache_read_tokens": 1066501383, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 9.583, "latency": 1447.794, "stderr": 2.148, "cost_per_test": 10.222944, "token_totals": { "input_tokens": 1143265902, "output_tokens": 13823660, "reasoning_tokens": null, "cache_read_tokens": 1089433052, "cache_write_tokens": 53824786 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 9.167, "latency": 1741.944, "stderr": 2.439, "cost_per_test": 0.997491, "token_totals": { "input_tokens": 1090091574, "output_tokens": 12155839, "reasoning_tokens": 0, "cache_read_tokens": 1056014080, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "tencent/hy4-preview": { "accuracy": 9.167, "latency": 1764.664, "stderr": 1.998, "cost_per_test": 0.98378, "token_totals": { "input_tokens": 1072794048, "output_tokens": 13132219, "reasoning_tokens": 4467767, "cache_read_tokens": 1022096320, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "fireworks/ember-1": { "accuracy": 9.167, "latency": 2665.557, "stderr": 2.439, "cost_per_test": 3.333599, "token_totals": { "input_tokens": 649364400, "output_tokens": 7643857, "reasoning_tokens": 2533467, "cache_read_tokens": 615821913, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 8.75, "latency": 452.101, "stderr": 2.148, "cost_per_test": 2.556743, "token_totals": { "input_tokens": 811642014, "output_tokens": 14269397, "reasoning_tokens": 3601488, "cache_read_tokens": 753832806, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 8.333, "latency": 1014.028, "stderr": 2.148, "cost_per_test": 0.059087, "token_totals": { "input_tokens": 744084543, "output_tokens": 7335131, "reasoning_tokens": 2960038, "cache_read_tokens": 723421312, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 8.333, "latency": 2558.175, "stderr": 1.998, "cost_per_test": 4.302903, "token_totals": { "input_tokens": 1456140757, "output_tokens": 16324375, "reasoning_tokens": null, "cache_read_tokens": 1398312225, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 7.5, "latency": 1313.321, "stderr": 2.148, "cost_per_test": 0.166206, "token_totals": { "input_tokens": 805739429, "output_tokens": 9454727, "reasoning_tokens": 4381477, "cache_read_tokens": 785343488, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 6.667, "latency": 579.44, "stderr": 1.832, "cost_per_test": 0.173471, "token_totals": { "input_tokens": 721200533, "output_tokens": 8383826, "reasoning_tokens": 4048611, "cache_read_tokens": 699334016, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 6.667, "latency": 3338.269, "stderr": 1.646, "cost_per_test": 23.669537, "token_totals": { "input_tokens": 2757683085, "output_tokens": 27559556, "reasoning_tokens": 6673656, "cache_read_tokens": 2623329853, "cache_write_tokens": 134339560 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 6.667, "latency": 6749.446, "stderr": 1.174, "cost_per_test": 46.211522, "token_totals": { "input_tokens": 2140493271, "output_tokens": 25042909, "reasoning_tokens": 9339413, "cache_read_tokens": 1833706187, "cache_write_tokens": 306775910 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 5.417, "latency": 1489.086, "stderr": 1.174, "cost_per_test": 26.155899, "token_totals": { "input_tokens": 1119192487, "output_tokens": 10562148, "reasoning_tokens": 3650237, "cache_read_tokens": 1049116892, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 5.417, "latency": 2594.18, "stderr": 1.755, "cost_per_test": 3.933528, "token_totals": { "input_tokens": 1310994009, "output_tokens": 13514557, "reasoning_tokens": 5869694, "cache_read_tokens": 1274733716, "cache_write_tokens": 36236558 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 5.0, "latency": 2331.577, "stderr": 1.646, "cost_per_test": 8.953859, "token_totals": { "input_tokens": 3340913248, "output_tokens": 20212831, "reasoning_tokens": null, "cache_read_tokens": 3252147720, "cache_write_tokens": 88748346 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 4.167, "latency": 1351.134, "stderr": 1.646, "cost_per_test": 1.460248, "token_totals": { "input_tokens": 1114902164, "output_tokens": 8397929, "reasoning_tokens": null, "cache_read_tokens": 1070555232, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 3.75, "latency": 401.522, "stderr": 1.431, "cost_per_test": 0.737465, "token_totals": { "input_tokens": 321058309, "output_tokens": 4167657, "reasoning_tokens": 272693, "cache_read_tokens": 303855232, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 3.75, "latency": 3175.128, "stderr": 1.469, "cost_per_test": 21.37949, "token_totals": { "input_tokens": 2462576678, "output_tokens": 38472452, "reasoning_tokens": 24925173, "cache_read_tokens": 2286257168, "cache_write_tokens": 176308820 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 2.917, "latency": 1051.83, "stderr": 1.362, "cost_per_test": 0.303552, "token_totals": { "input_tokens": 1460813994, "output_tokens": 12095727, "reasoning_tokens": 6619071, "cache_read_tokens": 1422964599, "cache_write_tokens": 37829190 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 2.917, "latency": 3464.426, "stderr": 1.362, "cost_per_test": 16.612583, "token_totals": { "input_tokens": 4750998856, "output_tokens": 53560846, "reasoning_tokens": 35373065, "cache_read_tokens": 4582495471, "cache_write_tokens": 168486127 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 2.5, "latency": 478.414, "stderr": 0.833, "cost_per_test": 2.116049, "token_totals": { "input_tokens": 606812300, "output_tokens": 8678382, "reasoning_tokens": 3806922, "cache_read_tokens": 543998500, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 2.5, "latency": 1370.296, "stderr": 0.833, "cost_per_test": 10.374554, "token_totals": { "input_tokens": 1221154582, "output_tokens": 10565145, "reasoning_tokens": 2786144, "cache_read_tokens": 1150617952, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 2.083, "latency": 544.226, "stderr": 0.833, "cost_per_test": 0.107095, "token_totals": { "input_tokens": 224714337, "output_tokens": 3198839, "reasoning_tokens": 0, "cache_read_tokens": 203250560, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling": { "accuracy": 2.083, "latency": 1103.692, "stderr": 0.833, "cost_per_test": 1.252023, "token_totals": { "input_tokens": 575866616, "output_tokens": 7829496, "reasoning_tokens": null, "cache_read_tokens": 552512512, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-sol": { "accuracy": 1.667, "latency": 820.097, "stderr": 0.823, "cost_per_test": 3.358015, "token_totals": { "input_tokens": 823114531, "output_tokens": 7639311, "reasoning_tokens": 3222223, "cache_read_tokens": 791317282, "cache_write_tokens": 31785134 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 1.667, "latency": 1870.966, "stderr": 0.833, "cost_per_test": 0.174542, "token_totals": { "input_tokens": 201049757, "output_tokens": 5401207, "reasoning_tokens": null, "cache_read_tokens": 191047296, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 1.25, "latency": 656.357, "stderr": 0.833, "cost_per_test": 0.383981, "token_totals": { "input_tokens": 993049543, "output_tokens": 9388666, "reasoning_tokens": 2979345, "cache_read_tokens": 909992290, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 1.25, "latency": 2113.923, "stderr": 0.716, "cost_per_test": 3.394654, "token_totals": { "input_tokens": 4127341953, "output_tokens": 54990908, "reasoning_tokens": 37472036, "cache_read_tokens": 4002588737, "cache_write_tokens": 124738510 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.833, "latency": 1024.886, "stderr": 0.833, "cost_per_test": 3.202399, "token_totals": { "input_tokens": 802970365, "output_tokens": 9301238, "reasoning_tokens": 2973597, "cache_read_tokens": 740704262, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 131.54, "stderr": 0.0, "cost_per_test": 0.192614, "token_totals": { "input_tokens": 159575776, "output_tokens": 1621389, "reasoning_tokens": 1524161, "cache_read_tokens": 55653173, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 436.69, "stderr": 0.0, "cost_per_test": 1.149757, "token_totals": { "input_tokens": 230732445, "output_tokens": 1350000, "reasoning_tokens": 503310, "cache_read_tokens": 204622317, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 681.27, "stderr": 0.0, "cost_per_test": 1.001493, "token_totals": { "input_tokens": 440584578, "output_tokens": 6595483, "reasoning_tokens": 5276888, "cache_read_tokens": 389590016, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null } }, "terminal_bench_4": { "anthropic/claude-opus-5-5": { "accuracy": 65.152, "latency": 3810.852, "stderr": 0.0, "cost_per_test": 13.198632, "token_totals": { "input_tokens": 1850938631, "output_tokens": 14913437, "reasoning_tokens": 10690470, "cache_read_tokens": 1824613192, "cache_write_tokens": 26310456 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 64.141, "latency": 4928.927, "stderr": 1.01, "cost_per_test": 16.513006, "token_totals": { "input_tokens": 3839535071, "output_tokens": 23509474, "reasoning_tokens": 17732859, "cache_read_tokens": 3801766853, "cache_write_tokens": 37747686 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 59.596, "latency": 2142.082, "stderr": 4.403, "cost_per_test": 9.584259, "token_totals": { "input_tokens": 193308076, "output_tokens": 5366501, "reasoning_tokens": 3845988, "cache_read_tokens": 184898940, "cache_write_tokens": 8367663 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 58.081, "latency": 3623.762, "stderr": 3.312, "cost_per_test": 17.175337, "token_totals": { "input_tokens": 1230443744, "output_tokens": 11653788, "reasoning_tokens": 7824002, "cache_read_tokens": 1208940872, "cache_write_tokens": 21489853 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 57.576, "latency": 3214.793, "stderr": 2.314, "cost_per_test": 17.63994, "token_totals": { "input_tokens": 1088824555, "output_tokens": 10645668, "reasoning_tokens": 6605548, "cache_read_tokens": 895783049, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 55.051, "latency": 2711.667, "stderr": 1.821, "cost_per_test": 1.72496, "token_totals": { "input_tokens": 182186678, "output_tokens": 6309120, "reasoning_tokens": 4640611, "cache_read_tokens": 172615332, "cache_write_tokens": 9543063 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 53.535, "latency": 3995.725, "stderr": 1.336, "cost_per_test": 18.596913, "token_totals": { "input_tokens": 1552207398, "output_tokens": 12957487, "reasoning_tokens": 8759246, "cache_read_tokens": 1530055510, "cache_write_tokens": 22137527 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 44.444, "latency": 2096.128, "stderr": 3.535, "cost_per_test": 5.79322, "token_totals": { "input_tokens": 903418096, "output_tokens": 6801174, "reasoning_tokens": 4146060, "cache_read_tokens": 891860011, "cache_write_tokens": 11430701 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 41.414, "latency": 4093.25, "stderr": 1.336, "cost_per_test": 30.339756, "token_totals": { "input_tokens": 1290713521, "output_tokens": 13036618, "reasoning_tokens": 9187742, "cache_read_tokens": 1266305087, "cache_write_tokens": 24396180 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 38.889, "latency": 4952.947, "stderr": 1.821, "cost_per_test": 9.37436, "token_totals": { "input_tokens": 2068794376, "output_tokens": 12825626, "reasoning_tokens": 9581216, "cache_read_tokens": 2047400960, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 37.879, "latency": 2026.072, "stderr": 0.0, "cost_per_test": 7.983597, "token_totals": { "input_tokens": 687501203, "output_tokens": 4776552, "reasoning_tokens": 2335318, "cache_read_tokens": 676679299, "cache_write_tokens": 10768374 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 35.354, "latency": 3059.553, "stderr": 2.201, "cost_per_test": 3.048418, "token_totals": { "input_tokens": 2927140508, "output_tokens": 20924574, "reasoning_tokens": 15966749, "cache_read_tokens": 2898581630, "cache_write_tokens": 28540965 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 34.343, "latency": 8900.152, "stderr": 3.945, "cost_per_test": 10.633329, "token_totals": { "input_tokens": 2221841269, "output_tokens": 16292661, "reasoning_tokens": 12333902, "cache_read_tokens": 2194079317, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "stepfun/step-5-preview": { "accuracy": 31.818, "latency": 7066.968, "stderr": 2.624, "cost_per_test": 3.113224, "token_totals": { "input_tokens": 2678379838, "output_tokens": 15864425, "reasoning_tokens": 0, "cache_read_tokens": 2648148395, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 31.313, "latency": 9720.048, "stderr": 3.072, "cost_per_test": 0.497505, "token_totals": { "input_tokens": 1740494990, "output_tokens": 15117928, "reasoning_tokens": 12015515, "cache_read_tokens": 1715650773, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 28.788, "latency": 3020.945, "stderr": 2.314, "cost_per_test": 18.091599, "token_totals": { "input_tokens": 1196356668, "output_tokens": 10523480, "reasoning_tokens": 6908182, "cache_read_tokens": 1149167019, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 24.747, "latency": 3176.371, "stderr": 0.505, "cost_per_test": 6.654565, "token_totals": { "input_tokens": 2125885043, "output_tokens": 14326152, "reasoning_tokens": 9822430, "cache_read_tokens": 2071855568, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 24.242, "latency": 8712.24, "stderr": 1.515, "cost_per_test": 0.216275, "token_totals": { "input_tokens": 2175333414, "output_tokens": 16125095, "reasoning_tokens": 12753311, "cache_read_tokens": 2150748075, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 23.232, "latency": 4283.164, "stderr": 1.336, "cost_per_test": 17.140717, "token_totals": { "input_tokens": 1116481668, "output_tokens": 16666742, "reasoning_tokens": 12858147, "cache_read_tokens": 1089283073, "cache_write_tokens": 27187396 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 22.727, "latency": 1888.568, "stderr": 2.314, "cost_per_test": 5.602118, "token_totals": { "input_tokens": 895160347, "output_tokens": 6238685, "reasoning_tokens": 3792707, "cache_read_tokens": 882387440, "cache_write_tokens": 12724594 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 22.727, "latency": 5070.406, "stderr": 0.875, "cost_per_test": 7.897179, "token_totals": { "input_tokens": 2100085922, "output_tokens": 17469108, "reasoning_tokens": 0, "cache_read_tokens": 1973708117, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 19.697, "latency": 2902.323, "stderr": 1.75, "cost_per_test": 0.499473, "token_totals": { "input_tokens": 2065521020, "output_tokens": 15407576, "reasoning_tokens": 12058588, "cache_read_tokens": 2058436005, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "fireworks/ember-1": { "accuracy": 19.697, "latency": 7308.779, "stderr": 3.154, "cost_per_test": 18.855717, "token_totals": { "input_tokens": 2890325482, "output_tokens": 13555938, "reasoning_tokens": 9749402, "cache_read_tokens": 2825866001, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 19.192, "latency": 6464.939, "stderr": 2.525, "cost_per_test": 8.774327, "token_totals": { "input_tokens": 2421985888, "output_tokens": 12571141, "reasoning_tokens": 7735968, "cache_read_tokens": 2331968020, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 18.687, "latency": 6368.24, "stderr": 3.535, "cost_per_test": 0.766472, "token_totals": { "input_tokens": 1973117830, "output_tokens": 15178104, "reasoning_tokens": 11818307, "cache_read_tokens": 1966243579, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.6": { "accuracy": 17.172, "latency": 1766.709, "stderr": 1.336, "cost_per_test": 5.071115, "token_totals": { "input_tokens": 549004132, "output_tokens": 5932401, "reasoning_tokens": 3375325, "cache_read_tokens": 532606037, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 17.172, "latency": 11339.786, "stderr": 0.505, "cost_per_test": 12.020926, "token_totals": { "input_tokens": 1706639152, "output_tokens": 12890103, "reasoning_tokens": 9374567, "cache_read_tokens": 1674032555, "cache_write_tokens": 31590377 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 14.141, "latency": 5538.07, "stderr": 2.02, "cost_per_test": 3.31413, "token_totals": { "input_tokens": 3602834705, "output_tokens": 11726281, "reasoning_tokens": 6905173, "cache_read_tokens": 3592041771, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 13.636, "latency": 1965.568, "stderr": 1.515, "cost_per_test": 0.354726, "token_totals": { "input_tokens": 1004162895, "output_tokens": 10957435, "reasoning_tokens": 7328334, "cache_read_tokens": 988222853, "cache_write_tokens": 15670624 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 12.121, "latency": 5031.11, "stderr": 3.03, "cost_per_test": 9.022683, "token_totals": { "input_tokens": 1574790554, "output_tokens": 10048068, "reasoning_tokens": 4117890, "cache_read_tokens": 1364480957, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 11.616, "latency": 2155.227, "stderr": 1.01, "cost_per_test": 0.731646, "token_totals": { "input_tokens": 1246925576, "output_tokens": 6795331, "reasoning_tokens": 3907781, "cache_read_tokens": 1232687204, "cache_write_tokens": 14195105 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 11.111, "latency": 5273.356, "stderr": 1.336, "cost_per_test": 3.378832, "token_totals": { "input_tokens": 3640330041, "output_tokens": 12278961, "reasoning_tokens": 7202675, "cache_read_tokens": 3629198635, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 10.606, "latency": 7112.43, "stderr": 1.75, "cost_per_test": 7.036467, "token_totals": { "input_tokens": 2197547446, "output_tokens": 13469736, "reasoning_tokens": 8997954, "cache_read_tokens": 2127067138, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 9.596, "latency": 6299.956, "stderr": 1.01, "cost_per_test": 26.325556, "token_totals": { "input_tokens": 4493866639, "output_tokens": 20598738, "reasoning_tokens": 14979244, "cache_read_tokens": 4453112934, "cache_write_tokens": 40727318 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 8.586, "latency": 2030.582, "stderr": 1.336, "cost_per_test": 5.79204, "token_totals": { "input_tokens": 654188164, "output_tokens": 5528209, "reasoning_tokens": 1256748, "cache_read_tokens": 634584149, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 8.081, "latency": 7798.686, "stderr": 1.336, "cost_per_test": 2.09742, "token_totals": { "input_tokens": 2695499187, "output_tokens": 14630101, "reasoning_tokens": 10474030, "cache_read_tokens": 2628586667, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 6.061, "latency": 4286.824, "stderr": 1.515, "cost_per_test": 5.981757, "token_totals": { "input_tokens": 1224247223, "output_tokens": 8938622, "reasoning_tokens": 5695714, "cache_read_tokens": 1127424034, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 6.061, "latency": 4705.164, "stderr": 1.515, "cost_per_test": 4.135537, "token_totals": { "input_tokens": 1413112583, "output_tokens": 9461885, "reasoning_tokens": 4674885, "cache_read_tokens": 1394234793, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 2.525, "latency": 1124.176, "stderr": 0.505, "cost_per_test": 4.111957, "token_totals": { "input_tokens": 521456492, "output_tokens": 3925593, "reasoning_tokens": 2337576, "cache_read_tokens": 487469934, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 2.525, "latency": 1780.682, "stderr": 1.01, "cost_per_test": 1.433702, "token_totals": { "input_tokens": 474199870, "output_tokens": 8650265, "reasoning_tokens": 6776184, "cache_read_tokens": 444372992, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 1.01, "latency": 4582.924, "stderr": 1.01, "cost_per_test": 7.229497, "token_totals": { "input_tokens": 3308478889, "output_tokens": 12669342, "reasoning_tokens": 2749857, "cache_read_tokens": 3254206054, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.505, "latency": 1371.962, "stderr": 0.505, "cost_per_test": 2.510486, "token_totals": { "input_tokens": 787734849, "output_tokens": 5705014, "reasoning_tokens": 0, "cache_read_tokens": 777286869, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.505, "latency": 7622.037, "stderr": 0.505, "cost_per_test": 0.23303, "token_totals": { "input_tokens": 594781184, "output_tokens": 7209202, "reasoning_tokens": 4359700, "cache_read_tokens": 578632021, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 705.115, "stderr": 0.0, "cost_per_test": 0.321679, "token_totals": { "input_tokens": 258225964, "output_tokens": 3667960, "reasoning_tokens": 0, "cache_read_tokens": 252660395, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 894.746, "stderr": 0.0, "cost_per_test": 0.369771, "token_totals": { "input_tokens": 261322790, "output_tokens": 2316679, "reasoning_tokens": 2220085, "cache_read_tokens": 164428874, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "tax_agent_bench": { "anthropic/claude-fable-5-1": { "accuracy": 77.642, "latency": 1714.017, "stderr": 2.835, "cost_per_test": 13.180073, "token_totals": { "input_tokens": 449166918, "output_tokens": 28237610, "reasoning_tokens": 7224, "cache_read_tokens": 365930262, "cache_write_tokens": 83209817 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 76.231, "latency": 985.634, "stderr": 2.874, "cost_per_test": 3.473645, "token_totals": { "input_tokens": 285519334, "output_tokens": 14493886, "reasoning_tokens": 6146392, "cache_read_tokens": 200405700, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 75.06, "latency": 1008.306, "stderr": 2.956, "cost_per_test": 5.136554, "token_totals": { "input_tokens": 322077193, "output_tokens": 16761490, "reasoning_tokens": 45865, "cache_read_tokens": 250544336, "cache_write_tokens": 71504899 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 73.395, "latency": 3560.542, "stderr": 2.981, "cost_per_test": 9.245466, "token_totals": { "input_tokens": 1150865417, "output_tokens": 108029684, "reasoning_tokens": 94199819, "cache_read_tokens": 954147139, "cache_write_tokens": 196666034 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 73.089, "latency": 3052.932, "stderr": 2.973, "cost_per_test": 1.80097, "token_totals": { "input_tokens": 229302538, "output_tokens": 47417212, "reasoning_tokens": 1131, "cache_read_tokens": 154199490, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 72.441, "latency": 248.818, "stderr": 2.879, "cost_per_test": 0.389917, "token_totals": { "input_tokens": 142091226, "output_tokens": 6095824, "reasoning_tokens": 3452149, "cache_read_tokens": 116606720, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 71.934, "latency": 505.365, "stderr": 2.917, "cost_per_test": 0.319558, "token_totals": { "input_tokens": 95475291, "output_tokens": 4737687, "reasoning_tokens": 2609786, "cache_read_tokens": 70731373, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 70.788, "latency": 657.844, "stderr": 3.005, "cost_per_test": 0.978328, "token_totals": { "input_tokens": 139046016, "output_tokens": 7673212, "reasoning_tokens": 4482527, "cache_read_tokens": 90209280, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 70.499, "latency": 4012.548, "stderr": 3.145, "cost_per_test": 15.086184, "token_totals": { "input_tokens": 815304119, "output_tokens": 88000495, "reasoning_tokens": 76703031, "cache_read_tokens": 648981098, "cache_write_tokens": 166280701 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 69.822, "latency": 866.628, "stderr": 3.118, "cost_per_test": 6.696087, "token_totals": { "input_tokens": 104187949, "output_tokens": 14106474, "reasoning_tokens": 10318710, "cache_read_tokens": 62150339, "cache_write_tokens": 42027339 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 68.672, "latency": 2415.159, "stderr": 3.083, "cost_per_test": 2.790127, "token_totals": { "input_tokens": 444117224, "output_tokens": 14995598, "reasoning_tokens": 10717157, "cache_read_tokens": 381463574, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/ember-1": { "accuracy": 68.044, "latency": 1845.939, "stderr": 3.128, "cost_per_test": 2.972904, "token_totals": { "input_tokens": 638768422, "output_tokens": 13714742, "reasoning_tokens": 9058386, "cache_read_tokens": 574529181, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 67.955, "latency": 2681.175, "stderr": 3.123, "cost_per_test": 7.513965, "token_totals": { "input_tokens": 608937344, "output_tokens": 36375165, "reasoning_tokens": 31251074, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 66.771, "latency": 178.329, "stderr": 3.148, "cost_per_test": 0.857214, "token_totals": { "input_tokens": 262586790, "output_tokens": 7012701, "reasoning_tokens": 4789793, "cache_read_tokens": 208172664, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 65.96, "latency": 2436.106, "stderr": 3.181, "cost_per_test": 1.552558, "token_totals": { "input_tokens": 201221502, "output_tokens": 20553651, "reasoning_tokens": 16744468, "cache_read_tokens": 130986496, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.7": { "accuracy": 65.596, "latency": 996.469, "stderr": 3.238, "cost_per_test": 1.796539, "token_totals": { "input_tokens": 280409764, "output_tokens": 10821528, "reasoning_tokens": 6708987, "cache_read_tokens": 186011136, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 65.199, "latency": 3630.72, "stderr": 3.188, "cost_per_test": 4.829225, "token_totals": { "input_tokens": 608154133, "output_tokens": 46971256, "reasoning_tokens": 41166217, "cache_read_tokens": 508089082, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 64.939, "latency": 1547.695, "stderr": 3.214, "cost_per_test": 0.134306, "token_totals": { "input_tokens": 136588210, "output_tokens": 12521886, "reasoning_tokens": 8284595, "cache_read_tokens": 102894848, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 64.906, "latency": 479.2, "stderr": 3.174, "cost_per_test": 1.68964, "token_totals": { "input_tokens": 58571818, "output_tokens": 7476677, "reasoning_tokens": 5120560, "cache_read_tokens": 39455480, "cache_write_tokens": 19099407 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 64.323, "latency": 283.566, "stderr": 3.156, "cost_per_test": 0.473604, "token_totals": { "input_tokens": 84693340, "output_tokens": 3280682, "reasoning_tokens": 949587, "cache_read_tokens": 57450112, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 63.712, "latency": 2464.028, "stderr": 3.26, "cost_per_test": 0.585277, "token_totals": { "input_tokens": 181603577, "output_tokens": 28903691, "reasoning_tokens": 24008954, "cache_read_tokens": 139882560, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-astra": { "accuracy": 63.335, "latency": 900.586, "stderr": 3.147, "cost_per_test": 5.7963, "token_totals": { "input_tokens": 115812555, "output_tokens": 12032734, "reasoning_tokens": 9165100, "cache_read_tokens": 71528751, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 63.294, "latency": 3438.427, "stderr": 3.229, "cost_per_test": 2.72124, "token_totals": { "input_tokens": 561529891, "output_tokens": 26978904, "reasoning_tokens": null, "cache_read_tokens": 287912448, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 62.563, "latency": 337.343, "stderr": 3.136, "cost_per_test": 0.156388, "token_totals": { "input_tokens": 118557354, "output_tokens": 14769391, "reasoning_tokens": 11893686, "cache_read_tokens": 97366102, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 62.458, "latency": 349.076, "stderr": 3.155, "cost_per_test": 0.134662, "token_totals": { "input_tokens": 114496920, "output_tokens": 15566679, "reasoning_tokens": 12729723, "cache_read_tokens": 91970688, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 62.31, "latency": 1213.878, "stderr": 3.2, "cost_per_test": 1.323601, "token_totals": { "input_tokens": 149308416, "output_tokens": 12717430, "reasoning_tokens": 9488118, "cache_read_tokens": 103439338, "cache_write_tokens": 45805894 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 62.273, "latency": 1158.894, "stderr": 3.186, "cost_per_test": 1.794774, "token_totals": { "input_tokens": 187222743, "output_tokens": 21888011, "reasoning_tokens": 10753790, "cache_read_tokens": 148056139, "cache_write_tokens": 39133536 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 62.203, "latency": 2973.647, "stderr": 3.222, "cost_per_test": 1.164207, "token_totals": { "input_tokens": 782193467, "output_tokens": 143162719, "reasoning_tokens": 133330984, "cache_read_tokens": 613966554, "cache_write_tokens": 168173092 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 60.815, "latency": 2565.695, "stderr": 3.263, "cost_per_test": 0.465359, "token_totals": { "input_tokens": 585632572, "output_tokens": 50154883, "reasoning_tokens": 43456905, "cache_read_tokens": 483082516, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 59.904, "latency": 1038.928, "stderr": 3.13, "cost_per_test": 0.048982, "token_totals": { "input_tokens": 168296091, "output_tokens": 12132305, "reasoning_tokens": 7352234, "cache_read_tokens": 127944128, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "stepfun/step-5-preview": { "accuracy": 59.539, "latency": 1288.515, "stderr": 3.09, "cost_per_test": 0.55338, "token_totals": { "input_tokens": 311523716, "output_tokens": 16401077, "reasoning_tokens": 0, "cache_read_tokens": 262109696, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-luna": { "accuracy": 58.864, "latency": 1995.736, "stderr": 3.211, "cost_per_test": 0.190956, "token_totals": { "input_tokens": 449103207, "output_tokens": 40664272, "reasoning_tokens": 37199155, "cache_read_tokens": 368052701, "cache_write_tokens": 80918209 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 58.663, "latency": 1596.755, "stderr": 3.138, "cost_per_test": 0.729136, "token_totals": { "input_tokens": 172298347, "output_tokens": 21059828, "reasoning_tokens": 17608474, "cache_read_tokens": 133313152, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 58.499, "latency": 1244.635, "stderr": 3.132, "cost_per_test": 0.719152, "token_totals": { "input_tokens": 171099768, "output_tokens": 21311454, "reasoning_tokens": 17772506, "cache_read_tokens": 134364160, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 57.665, "latency": 87.24, "stderr": 3.313, "cost_per_test": 0.483978, "token_totals": { "input_tokens": 104490097, "output_tokens": 3925073, "reasoning_tokens": 2390046, "cache_read_tokens": 68715157, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 56.86, "latency": 119.905, "stderr": 2.318, "cost_per_test": 0.269212, "token_totals": { "input_tokens": 84100295, "output_tokens": 3814259, "reasoning_tokens": 1704398, "cache_read_tokens": 63070995, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 53.526, "latency": 233.001, "stderr": 3.302, "cost_per_test": 0.859679, "token_totals": { "input_tokens": 335113206, "output_tokens": 5873488, "reasoning_tokens": 4405887, "cache_read_tokens": 288602367, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 53.045, "latency": 1052.036, "stderr": 3.29, "cost_per_test": 2.277613, "token_totals": { "input_tokens": 386182808, "output_tokens": 18761287, "reasoning_tokens": 15767772, "cache_read_tokens": 316266542, "cache_write_tokens": 69503073 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 49.691, "latency": 314.575, "stderr": 3.248, "cost_per_test": 0.164063, "token_totals": { "input_tokens": 85169988, "output_tokens": 4820320, "reasoning_tokens": 176867, "cache_read_tokens": 22014292, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 43.516, "latency": 810.557, "stderr": 2.947, "cost_per_test": 0.353757, "token_totals": { "input_tokens": 147459631, "output_tokens": 7269937, "reasoning_tokens": null, "cache_read_tokens": 130876928, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 41.461, "latency": 385.252, "stderr": 2.871, "cost_per_test": 0.091767, "token_totals": { "input_tokens": 107926904, "output_tokens": 7430357, "reasoning_tokens": null, "cache_read_tokens": 98264704, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 37.842, "latency": 100.956, "stderr": 3.098, "cost_per_test": 0.301555, "token_totals": { "input_tokens": 33380072, "output_tokens": 1903235, "reasoning_tokens": 1460348, "cache_read_tokens": 17443775, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 36.586, "latency": 1006.185, "stderr": 3.095, "cost_per_test": 0.806171, "token_totals": { "input_tokens": 201253946, "output_tokens": 26480732, "reasoning_tokens": 25245242, "cache_read_tokens": 169648384, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 34.787, "latency": 376.514, "stderr": 2.621, "cost_per_test": 0.040042, "token_totals": { "input_tokens": 66417509, "output_tokens": 2436638, "reasoning_tokens": 1177368, "cache_read_tokens": 53971712, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "inception/mercury-2.5": { "accuracy": 12.41, "latency": 29.67, "stderr": 1.805, "cost_per_test": 0.023633, "token_totals": { "input_tokens": 43610163, "output_tokens": 1545275, "reasoning_tokens": 1248847, "cache_read_tokens": 29554557, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } } } }