{ "metadata": { "benchmark": "Vals Index", "slug": "vals_index_v2", "description": "A single measure of AI's potential economic impact — agentic model performance across finance, coding, and legal tasks, weighted by each sector's share of U.S. GDP.", "benchmark_id": "vals_index_v2", "family": "vals_index", "version": "2", "updated": "2026-09-27", "dataset_type": "private", "industry": "index", "tasks": { "overall": "Overall", "finance_agent": "Finance Agent v2", "emb": "EMB", "terminal_bench_2_1": "Terminal-Bench 2.1", "vibe_code_bench": "Vibe Code Bench", "code_migration": "Code Migration", "legal_research": "Legal Research", "legal_agent_benchmark": "HLAB" }, "models": [ "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.7-plus", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "ant/ling-3.0-flash-2607", "ant/ling-3.0-flash-af-rc3", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "fireworks/nemotron-lightning-3p5-30b-a3b", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-4.7", "inception/mercury-2.5", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/mistral-medium-3.5", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.2", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": true, "partner": false, "total_models": 66 }, "tasks": { "overall": { "anthropic/claude-opus-5-5": { "accuracy": 69.689, "latency": 4320.731, "stderr": 0.937, "cost_per_test": 32.766011, "token_totals": { "input_tokens": 26111736437, "output_tokens": 390179801, "reasoning_tokens": null, "cache_read_tokens": 24998910961, "cache_write_tokens": 936264521 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 69.219, "latency": 4207.124, "stderr": 0.962, "cost_per_test": 20.803442, "token_totals": { "input_tokens": 35835336175, "output_tokens": 480466610, "reasoning_tokens": null, "cache_read_tokens": 34417379870, "cache_write_tokens": 1239930755 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 68.825, "latency": 4576.315, "stderr": 1.077, "cost_per_test": 28.916238, "token_totals": { "input_tokens": 17294977546, "output_tokens": 163782272, "reasoning_tokens": null, "cache_read_tokens": 15077827193, "cache_write_tokens": 1910172076 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 67.213, "latency": 3348.565, "stderr": 0.978, "cost_per_test": 18.810212, "token_totals": { "input_tokens": 12870034577, "output_tokens": 118968302, "reasoning_tokens": null, "cache_read_tokens": 12219125012, "cache_write_tokens": 516373519 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 66.608, "latency": 1510.529, "stderr": 1.093, "cost_per_test": 19.088776, "token_totals": { "input_tokens": 3642278125, "output_tokens": 78432657, "reasoning_tokens": 46744595, "cache_read_tokens": 3200902646, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 66.036, "latency": 2271.1, "stderr": 1.028, "cost_per_test": 28.725071, "token_totals": { "input_tokens": 9053399672, "output_tokens": 98649749, "reasoning_tokens": null, "cache_read_tokens": 8642799312, "cache_write_tokens": 351068223 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 64.526, "latency": 1230.411, "stderr": 1.23, "cost_per_test": 3.404111, "token_totals": { "input_tokens": 9211027008, "output_tokens": 77477083, "reasoning_tokens": 36960671, "cache_read_tokens": 8846511178, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 63.709, "latency": 1943.907, "stderr": 1.06, "cost_per_test": 14.205832, "token_totals": { "input_tokens": 5191613777, "output_tokens": 105483861, "reasoning_tokens": null, "cache_read_tokens": 4606830773, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 62.569, "latency": 1593.734, "stderr": 0.954, "cost_per_test": 7.564103, "token_totals": { "input_tokens": 7733082111, "output_tokens": 80591178, "reasoning_tokens": 54480477, "cache_read_tokens": 7297148411, "cache_write_tokens": 402998059 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 62.252, "latency": 2692.685, "stderr": 1.009, "cost_per_test": 5.391952, "token_totals": { "input_tokens": 10652216618, "output_tokens": 99723680, "reasoning_tokens": 57266178, "cache_read_tokens": 9512667147, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 60.908, "latency": 2182.138, "stderr": 1.027, "cost_per_test": 12.66685, "token_totals": { "input_tokens": 6892446411, "output_tokens": 99206482, "reasoning_tokens": null, "cache_read_tokens": 6474229382, "cache_write_tokens": 364023333 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 60.31, "latency": 1276.983, "stderr": 1.076, "cost_per_test": 2.899528, "token_totals": { "input_tokens": 5981055926, "output_tokens": 62215005, "reasoning_tokens": 30184709, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 60.215, "latency": 1975.967, "stderr": 1.079, "cost_per_test": 11.161883, "token_totals": { "input_tokens": 5903398282, "output_tokens": 95277104, "reasoning_tokens": 54595046, "cache_read_tokens": 5022990464, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 59.881, "latency": 1396.334, "stderr": 1.087, "cost_per_test": 0.778845, "token_totals": { "input_tokens": 7718348043, "output_tokens": 172065971, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 59.61, "latency": 2745.876, "stderr": 1.147, "cost_per_test": 11.740516, "token_totals": { "input_tokens": 22035020760, "output_tokens": 126592712, "reasoning_tokens": null, "cache_read_tokens": 21618249208, "cache_write_tokens": 327717709 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 59.588, "latency": 1730.183, "stderr": 1.331, "cost_per_test": 5.178734, "token_totals": { "input_tokens": 5097991442, "output_tokens": 124108261, "reasoning_tokens": 92127311, "cache_read_tokens": 4467139215, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 59.584, "latency": 2701.771, "stderr": 1.133, "cost_per_test": 0.19671, "token_totals": { "input_tokens": 9275483161, "output_tokens": 98366151, "reasoning_tokens": 56687166, "cache_read_tokens": 8744903613, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 59.47, "latency": 3139.768, "stderr": 1.177, "cost_per_test": 0.38918, "token_totals": { "input_tokens": 6536770815, "output_tokens": 94727410, "reasoning_tokens": 62090256, "cache_read_tokens": 6114753728, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 59.307, "latency": 2592.928, "stderr": 1.065, "cost_per_test": 4.169542, "token_totals": { "input_tokens": 7711961831, "output_tokens": 78223196, "reasoning_tokens": 31671638, "cache_read_tokens": 6872181788, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 59.167, "latency": 2283.847, "stderr": 1.205, "cost_per_test": 4.335319, "token_totals": { "input_tokens": 3768589157, "output_tokens": 91956684, "reasoning_tokens": 50438438, "cache_read_tokens": 3338271659, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 58.45, "latency": 1590.457, "stderr": 1.107, "cost_per_test": 0.418873, "token_totals": { "input_tokens": 8367574348, "output_tokens": 152362466, "reasoning_tokens": 124414611, "cache_read_tokens": 7932356337, "cache_write_tokens": 396158822 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 57.86, "latency": 1534.071, "stderr": 1.16, "cost_per_test": 0.303291, "token_totals": { "input_tokens": 7930348156, "output_tokens": 114241342, "reasoning_tokens": 85006692, "cache_read_tokens": 7650798336, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 57.813, "latency": 4173.697, "stderr": 1.058, "cost_per_test": 6.467826, "token_totals": { "input_tokens": 5873738338, "output_tokens": 73291657, "reasoning_tokens": 42547737, "cache_read_tokens": 2030185983, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.5": { "accuracy": 57.411, "latency": 1251.251, "stderr": 1.151, "cost_per_test": 6.142389, "token_totals": { "input_tokens": 2253903341, "output_tokens": 56691895, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 57.053, "latency": 1009.228, "stderr": 1.117, "cost_per_test": 1.661592, "token_totals": { "input_tokens": 3696286199, "output_tokens": 53119242, "reasoning_tokens": 21288023, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 56.97, "latency": 3988.986, "stderr": 1.349, "cost_per_test": 6.73155, "token_totals": { "input_tokens": 9834508062, "output_tokens": 153840469, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 56.114, "latency": 1705.956, "stderr": 1.214, "cost_per_test": 12.760253, "token_totals": { "input_tokens": 7918355680, "output_tokens": 73331613, "reasoning_tokens": null, "cache_read_tokens": 7479074091, "cache_write_tokens": 369806577 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 55.402, "latency": 2871.238, "stderr": 1.248, "cost_per_test": 1.280885, "token_totals": { "input_tokens": 7896397101, "output_tokens": 138896099, "reasoning_tokens": 101793186, "cache_read_tokens": 7512594048, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 55.354, "latency": 1328.048, "stderr": 1.092, "cost_per_test": 3.02952, "token_totals": { "input_tokens": 5329006331, "output_tokens": 64363969, "reasoning_tokens": 30398137, "cache_read_tokens": 4648170977, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 54.754, "latency": 827.803, "stderr": 1.148, "cost_per_test": 1.250502, "token_totals": { "input_tokens": 3075412734, "output_tokens": 49060986, "reasoning_tokens": 24744202, "cache_read_tokens": 2776673825, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 53.568, "latency": 2094.581, "stderr": 1.21, "cost_per_test": 0.839765, "token_totals": { "input_tokens": 12458821976, "output_tokens": 105942050, "reasoning_tokens": 73608510, "cache_read_tokens": 11876848853, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 53.122, "latency": 2527.312, "stderr": 1.184, "cost_per_test": 4.290631, "token_totals": { "input_tokens": 4897387298, "output_tokens": 72578822, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 53.079, "latency": 661.463, "stderr": 1.122, "cost_per_test": 2.914646, "token_totals": { "input_tokens": 5439070661, "output_tokens": 73417311, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 52.368, "latency": 3498.495, "stderr": 1.136, "cost_per_test": 3.282301, "token_totals": { "input_tokens": 14318465577, "output_tokens": 123476034, "reasoning_tokens": 85438326, "cache_read_tokens": 13443117482, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 51.844, "latency": 5445.777, "stderr": 1.288, "cost_per_test": 3.996716, "token_totals": { "input_tokens": 5366385489, "output_tokens": 113817573, "reasoning_tokens": 82447375, "cache_read_tokens": 4952847874, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 51.528, "latency": 837.691, "stderr": 1.317, "cost_per_test": 1.955641, "token_totals": { "input_tokens": 1572959756, "output_tokens": 48109129, "reasoning_tokens": 15122055, "cache_read_tokens": 1293726570, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 50.593, "latency": 2085.722, "stderr": 1.235, "cost_per_test": 8.404689, "token_totals": { "input_tokens": 9395598561, "output_tokens": 77934688, "reasoning_tokens": null, "cache_read_tokens": 9032066622, "cache_write_tokens": 320789955 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 48.485, "latency": 3634.583, "stderr": 1.454, "cost_per_test": 8.479089, "token_totals": { "input_tokens": 7989118693, "output_tokens": 149103691, "reasoning_tokens": 110337354, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 47.216, "latency": 4266.783, "stderr": 1.445, "cost_per_test": 0.992509, "token_totals": { "input_tokens": 28466247263, "output_tokens": 149734128, "reasoning_tokens": null, "cache_read_tokens": 27854888107, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 44.769, "latency": 1781.791, "stderr": 1.078, "cost_per_test": 2.850897, "token_totals": { "input_tokens": 1204062069, "output_tokens": 30092870, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 43.465, "latency": 2392.268, "stderr": 1.174, "cost_per_test": 1.849897, "token_totals": { "input_tokens": 6705030670, "output_tokens": 58025942, "reasoning_tokens": 11486, "cache_read_tokens": 6148606992, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 42.888, "latency": 1427.349, "stderr": 1.194, "cost_per_test": 0.940927, "token_totals": { "input_tokens": 4286018452, "output_tokens": 64454205, "reasoning_tokens": 35695726, "cache_read_tokens": 3973063859, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 42.719, "latency": 2114.745, "stderr": 1.198, "cost_per_test": 2.495943, "token_totals": { "input_tokens": 6310009185, "output_tokens": 54937051, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 41.903, "latency": 560.152, "stderr": 1.173, "cost_per_test": 1.944534, "token_totals": { "input_tokens": 2600449360, "output_tokens": 24148182, "reasoning_tokens": 14278256, "cache_read_tokens": 2184929887, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 40.974, "latency": 1350.372, "stderr": 1.377, "cost_per_test": 0.152141, "token_totals": { "input_tokens": 2508435735, "output_tokens": 39317378, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 39.91, "latency": 1092.973, "stderr": 1.312, "cost_per_test": 0.061288, "token_totals": { "input_tokens": 2780186533, "output_tokens": 40358894, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 39.632, "latency": 1742.074, "stderr": 1.435, "cost_per_test": 1.380101, "token_totals": { "input_tokens": 4796316300, "output_tokens": 178576215, "reasoning_tokens": 158816517, "cache_read_tokens": 4048558110, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 38.645, "latency": 1307.967, "stderr": 1.171, "cost_per_test": 0.439693, "token_totals": { "input_tokens": 1643910163, "output_tokens": 39049189, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 36.711, "latency": 461.093, "stderr": 1.065, "cost_per_test": 0.543466, "token_totals": { "input_tokens": 3393546561, "output_tokens": 35025139, "reasoning_tokens": 16840013, "cache_read_tokens": 2350435985, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling": { "accuracy": 34.102, "latency": 1176.629, "stderr": 1.132, "cost_per_test": 1.354489, "token_totals": { "input_tokens": 3804323111, "output_tokens": 59528073, "reasoning_tokens": null, "cache_read_tokens": 3483172924, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 32.997, "latency": 1229.587, "stderr": 1.276, "cost_per_test": 0.398254, "token_totals": { "input_tokens": 5549058010, "output_tokens": 55274234, "reasoning_tokens": 34624369, "cache_read_tokens": 4974306282, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 32.553, "latency": 1072.857, "stderr": 0.966, "cost_per_test": 0.109173, "token_totals": { "input_tokens": 3232017941, "output_tokens": 67666363, "reasoning_tokens": null, "cache_read_tokens": 2709221952, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 31.979, "latency": 1260.202, "stderr": 0.976, "cost_per_test": 2.618074, "token_totals": { "input_tokens": 2737393675, "output_tokens": 38741690, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 31.862, "latency": 1372.678, "stderr": 1.103, "cost_per_test": 0.281249, "token_totals": { "input_tokens": 2275513174, "output_tokens": 58895002, "reasoning_tokens": null, "cache_read_tokens": 2105815595, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 29.444, "latency": 348.217, "stderr": 0.983, "cost_per_test": 0.409423, "token_totals": { "input_tokens": 1702600857, "output_tokens": 31783858, "reasoning_tokens": 19424855, "cache_read_tokens": 1132293364, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 27.395, "latency": 1948.359, "stderr": 0.949, "cost_per_test": null, "token_totals": { "input_tokens": 4573577723, "output_tokens": 40658976, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 26.301, "latency": 2130.649, "stderr": 0.948, "cost_per_test": 0.511754, "token_totals": { "input_tokens": 2042074549, "output_tokens": 28860885, "reasoning_tokens": 9538508, "cache_read_tokens": 1815209193, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 24.556, "latency": 904.14, "stderr": 0.932, "cost_per_test": 0.87768, "token_totals": { "input_tokens": 2645081856, "output_tokens": 41972016, "reasoning_tokens": null, "cache_read_tokens": 1204649149, "cache_write_tokens": 20552360 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4.3": { "accuracy": 24.294, "latency": 338.369, "stderr": 0.813, "cost_per_test": 0.676033, "token_totals": { "input_tokens": 1163459787, "output_tokens": 33067164, "reasoning_tokens": 18210520, "cache_read_tokens": 859598709, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 22.898, "latency": 500.129, "stderr": 0.839, "cost_per_test": 0.697729, "token_totals": { "input_tokens": 2423312874, "output_tokens": 34166900, "reasoning_tokens": null, "cache_read_tokens": 2161493870, "cache_write_tokens": 243274515 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 21.699, "latency": 2982.634, "stderr": 0.783, "cost_per_test": 0.104406, "token_totals": { "input_tokens": 2521319020, "output_tokens": 48997437, "reasoning_tokens": null, "cache_read_tokens": 2069698382, "cache_write_tokens": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 17.947, "latency": 3446.487, "stderr": 0.687, "cost_per_test": 8.010266, "token_totals": { "input_tokens": 2751786234, "output_tokens": 53383576, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 17.552, "latency": 349.748, "stderr": 0.526, "cost_per_test": 0.569009, "token_totals": { "input_tokens": 8027608295, "output_tokens": 21303467, "reasoning_tokens": 13102246, "cache_read_tokens": 7499919053, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 15.457, "latency": 162.534, "stderr": 0.417, "cost_per_test": 0.136314, "token_totals": { "input_tokens": 675777414, "output_tokens": 12042817, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 12.952, "latency": 354.783, "stderr": 0.474, "cost_per_test": 0.300647, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 11.494, "latency": 1345.167, "stderr": 0.61, "cost_per_test": 0.09345, "token_totals": { "input_tokens": 3640709809, "output_tokens": 66857971, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "vibe_code_bench": { "anthropic/claude-sonnet-5-5": { "accuracy": 92.392, "latency": 3672.106, "stderr": 1.26, "cost_per_test": 31.250578, "token_totals": { "input_tokens": 5822927217, "output_tokens": 25491041, "reasoning_tokens": 11893204, "cache_read_tokens": 5760733058, "cache_write_tokens": 62167153 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "anthropic/claude-fable-5": { "accuracy": 90.352, "latency": 3711.026, "stderr": 2.097, "cost_per_test": 41.714694, "token_totals": { "input_tokens": 1628443000, "output_tokens": 8412174, "reasoning_tokens": 2231064, "cache_read_tokens": 1604184760, "cache_write_tokens": 24245069 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "anthropic/claude-opus-5-5": { "accuracy": 90.294, "latency": 5766.524, "stderr": 1.526, "cost_per_test": 57.922811, "token_totals": { "input_tokens": 5596583425, "output_tokens": 19722501, "reasoning_tokens": 7457724, "cache_read_tokens": 5539450465, "cache_write_tokens": 57102136 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "anthropic/claude-fable-5-1": { "accuracy": 90.263, "latency": 3459.994, "stderr": 1.566, "cost_per_test": 33.367507, "token_totals": { "input_tokens": 2858376432, "output_tokens": 11229337, "reasoning_tokens": 2013796, "cache_read_tokens": 2826345790, "cache_write_tokens": 32006248 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "openai/gpt-6-astra": { "accuracy": 89.594, "latency": 2344.127, "stderr": 2.174, "cost_per_test": 38.512323, "token_totals": { "input_tokens": 813962842, "output_tokens": 7672087, "reasoning_tokens": 3013580, "cache_read_tokens": 762745769, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "anthropic/claude-opus-5": { "accuracy": 88.404, "latency": 5301.22, "stderr": 3.001, "cost_per_test": 33.876235, "token_totals": { "input_tokens": 2505984639, "output_tokens": 9909555, "reasoning_tokens": 1547105, "cache_read_tokens": 2470549366, "cache_write_tokens": 35417005 }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "OpenHands" }, "openai/gpt-6-sol": { "accuracy": 87.824, "latency": 2336.393, "stderr": 2.526, "cost_per_test": 26.356012, "token_totals": { "input_tokens": 3192235437, "output_tokens": 6823472, "reasoning_tokens": 2395580, "cache_read_tokens": 3162891655, "cache_write_tokens": 29258099 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "grok/grok-4.7": { "accuracy": 86.175, "latency": 2147.065, "stderr": 2.181, "cost_per_test": 15.82706, "token_totals": { "input_tokens": 767137889, "output_tokens": 7923865, "reasoning_tokens": 3838088, "cache_read_tokens": 713123456, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "meta/muse_spark_1_3_max": { "accuracy": 85.856, "latency": 977.668, "stderr": 2.508, "cost_per_test": 2.543084, "token_totals": { "input_tokens": 651739493, "output_tokens": 3048548, "reasoning_tokens": 422854, "cache_read_tokens": 636796800, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "xiaomi/mimo-v2.6-pro": { "accuracy": 85.223, "latency": 3646.574, "stderr": 3.392, "cost_per_test": 1.043846, "token_totals": { "input_tokens": 1076595168, "output_tokens": 7478091, "reasoning_tokens": 2852217, "cache_read_tokens": 979676672, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands" }, "kimi/kimi-k3": { "accuracy": 84.963, "latency": 5202.941, "stderr": 2.69, "cost_per_test": 17.590352, "token_totals": { "input_tokens": 1395659476, "output_tokens": 5599446, "reasoning_tokens": 1497200, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "OpenHands" }, "deepseek/deepseek-v4.1-flash": { "accuracy": 84.739, "latency": 928.101, "stderr": 2.891, "cost_per_test": 0.407445, "token_totals": { "input_tokens": 1429431621, "output_tokens": 7186472, "reasoning_tokens": 1913924, "cache_read_tokens": 1418642812, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "meta/muse_spark_1_3": { "accuracy": 82.859, "latency": 773.797, "stderr": 2.901, "cost_per_test": 2.101762, "token_totals": { "input_tokens": 401837405, "output_tokens": 2713902, "reasoning_tokens": 370079, "cache_read_tokens": 371584301, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "anthropic/claude-opus-4-8": { "accuracy": 82.725, "latency": 4549.256, "stderr": 3.081, "cost_per_test": 26.879989, "token_totals": { "input_tokens": 1802936139, "output_tokens": 9844882, "reasoning_tokens": 3416693, "cache_read_tokens": 1781691366, "cache_write_tokens": 21227618 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 82.297, "latency": 3982.6, "stderr": 3.148, "cost_per_test": 0.356023, "token_totals": { "input_tokens": 1830028483, "output_tokens": 7746431, "reasoning_tokens": 2336919, "cache_read_tokens": 1820296320, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "openai/gpt-6-luna": { "accuracy": 81.649, "latency": 2219.515, "stderr": 3.377, "cost_per_test": 1.345703, "token_totals": { "input_tokens": 3186713261, "output_tokens": 9004550, "reasoning_tokens": 3632878, "cache_read_tokens": 3158117507, "cache_write_tokens": 28566327 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "anthropic/claude-sonnet-5": { "accuracy": 81.325, "latency": 4001.424, "stderr": 3.046, "cost_per_test": 25.38542, "token_totals": { "input_tokens": 5886886074, "output_tokens": 13734667, "reasoning_tokens": 5002326, "cache_read_tokens": 5849205195, "cache_write_tokens": 37648445 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "openai/gpt-5.6-sol": { "accuracy": 80.495, "latency": 2033.047, "stderr": 3.716, "cost_per_test": 33.403634, "token_totals": { "input_tokens": 85484634, "output_tokens": 3878003, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "meta/muse_spark_1_2": { "accuracy": 79.1, "latency": 1195.221, "stderr": 3.305, "cost_per_test": 1.529642, "token_totals": { "input_tokens": 342866386, "output_tokens": 2351182, "reasoning_tokens": 239369, "cache_read_tokens": 329175836, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "xiaomi/mimo-v2.6-flash": { "accuracy": 78.96, "latency": 3013.012, "stderr": 4.045, "cost_per_test": 0.563041, "token_totals": { "input_tokens": 1311689878, "output_tokens": 7947370, "reasoning_tokens": 3093850, "cache_read_tokens": 1157695808, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands" }, "google/gemini-3.8-flash": { "accuracy": 78.651, "latency": 519.104, "stderr": 3.884, "cost_per_test": 6.865141, "token_totals": { "input_tokens": 1149521706, "output_tokens": 6144190, "reasoning_tokens": 1439332, "cache_read_tokens": 1057116240, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "zai/glm-5.3": { "accuracy": 78.125, "latency": 3847.197, "stderr": 3.94, "cost_per_test": 12.450185, "token_totals": { "input_tokens": 1971471587, "output_tokens": 8173329, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "tencent/hy4-preview": { "accuracy": 77.478, "latency": 2493.104, "stderr": 4.035, "cost_per_test": 2.181516, "token_totals": { "input_tokens": 1648623995, "output_tokens": 6263219, "reasoning_tokens": 1671679, "cache_read_tokens": 1618107200, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": "OpenHands" }, "openai/gpt-5.6-luna": { "accuracy": 77.055, "latency": 1551.831, "stderr": 3.069, "cost_per_test": 0.726426, "token_totals": { "input_tokens": 54304210, "output_tokens": 3660213, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "grok/grok-4.6": { "accuracy": 76.239, "latency": 1530.145, "stderr": 3.824, "cost_per_test": 4.875513, "token_totals": { "input_tokens": 395097143, "output_tokens": 3804849, "reasoning_tokens": 775807, "cache_read_tokens": 379498496, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 74.736, "latency": 2241.572, "stderr": 3.442, "cost_per_test": 0.905592, "token_totals": { "input_tokens": 2153389585, "output_tokens": 7751888, "reasoning_tokens": 2696365, "cache_read_tokens": 2141888000, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "openai/gpt-5.6-terra": { "accuracy": 74.594, "latency": 1152.433, "stderr": 4.203, "cost_per_test": 7.887015, "token_totals": { "input_tokens": 1192383214, "output_tokens": 4865371, "reasoning_tokens": 1379403, "cache_read_tokens": 1138222288, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "meta/muse_spark_1_1": { "accuracy": 72.158, "latency": 615.428, "stderr": 3.327, "cost_per_test": 0.923283, "token_totals": { "input_tokens": 213516750, "output_tokens": 2218049, "reasoning_tokens": 189712, "cache_read_tokens": 182918822, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": "OpenHands" }, "anthropic/claude-opus-4-7": { "accuracy": 71.003, "latency": 2141.902, "stderr": 4.506, "cost_per_test": 21.407199, "token_totals": { "input_tokens": 1304465300, "output_tokens": 5421314, "reasoning_tokens": 277630, "cache_read_tokens": 1286180434, "cache_write_tokens": 18267241 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "google/gemini-3.7-flash": { "accuracy": 70.395, "latency": 1423.494, "stderr": 4.837, "cost_per_test": 4.827643, "token_totals": { "input_tokens": 911735026, "output_tokens": 5880420, "reasoning_tokens": 1013076, "cache_read_tokens": 866906330, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "openai/gpt-5.5": { "accuracy": 69.847, "latency": 1911.567, "stderr": 4.535, "cost_per_test": 16.661911, "token_totals": { "input_tokens": 92633221, "output_tokens": 3089898, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "grok/grok-4.5": { "accuracy": 69.002, "latency": 830.057, "stderr": 4.48, "cost_per_test": 4.102194, "token_totals": { "input_tokens": 264749332, "output_tokens": 3594207, "reasoning_tokens": 64753, "cache_read_tokens": 254635589, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "alibaba/qwen3.8-27b": { "accuracy": 64.85, "latency": 2619.407, "stderr": 4.825, "cost_per_test": 23.456372, "token_totals": { "input_tokens": 2292687899, "output_tokens": 8824879, "reasoning_tokens": 3211356, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "alibaba/qwen3.8-max": { "accuracy": 64.696, "latency": 10135.139, "stderr": 5.362, "cost_per_test": 8.238852, "token_totals": { "input_tokens": 1000088567, "output_tokens": 6067236, "reasoning_tokens": 2004745, "cache_read_tokens": 921541719, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "google/gemini-3.6-flash": { "accuracy": 64.005, "latency": 1523.201, "stderr": 4.29, "cost_per_test": 3.041993, "token_totals": { "input_tokens": 419948837, "output_tokens": 4065006, "reasoning_tokens": 715555, "cache_read_tokens": 376526772, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "zai/glm-5.2": { "accuracy": 63.961, "latency": 3748.167, "stderr": 4.795, "cost_per_test": 8.488514, "token_totals": { "input_tokens": 55483571, "output_tokens": 4081373, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "anthropic/claude-sonnet-4-6": { "accuracy": 51.476, "latency": 1572.118, "stderr": 4.645, "cost_per_test": 5.907892, "token_totals": { "input_tokens": 567692336, "output_tokens": 4210455, "reasoning_tokens": 442221, "cache_read_tokens": 557548000, "cache_write_tokens": 10135852 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": "OpenHands" }, "deepseek/deepseek-v4-pro": { "accuracy": 49.931, "latency": 3418.92, "stderr": 4.767, "cost_per_test": 2.205991, "token_totals": { "input_tokens": 1767336432, "output_tokens": 7905241, "reasoning_tokens": 2425211, "cache_read_tokens": 1758006109, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": "OpenHands" }, "google/gemini-3.5-flash": { "accuracy": 48.683, "latency": 893.074, "stderr": 4.727, "cost_per_test": 2.540385, "token_totals": { "input_tokens": 64968703, "output_tokens": 3285132, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 47.969, "latency": 2055.117, "stderr": 5.606, "cost_per_test": 1.191299, "token_totals": { "input_tokens": 1529114850, "output_tokens": 14510964, "reasoning_tokens": 8384750, "cache_read_tokens": 1305150933, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "alibaba/qwen3.7-max": { "accuracy": 47.671, "latency": 4689.627, "stderr": 4.63, "cost_per_test": 11.400322, "token_totals": { "input_tokens": 197229291, "output_tokens": 2354762, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "minimax/MiniMax-M3": { "accuracy": 47.566, "latency": 5073.23, "stderr": 5.442, "cost_per_test": 6.453899, "token_totals": { "input_tokens": 61631839, "output_tokens": 5567819, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": "OpenHands" }, "alibaba/qwen3.7-plus": { "accuracy": 46.391, "latency": 2231.75, "stderr": 4.614, "cost_per_test": 1.082348, "token_totals": { "input_tokens": 55322964, "output_tokens": 3391824, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "xiaomi/mimo-v2.5": { "accuracy": 42.174, "latency": 1580.348, "stderr": 4.568, "cost_per_test": 0.068395, "token_totals": { "input_tokens": 7646791, "output_tokens": 2710586, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands" }, "kimi/kimi-k2.6": { "accuracy": 37.891, "latency": 2967.768, "stderr": 4.912, "cost_per_test": 1.925351, "token_totals": { "input_tokens": 1291837961, "output_tokens": 3159307, "reasoning_tokens": 11486, "cache_read_tokens": 1269484743, "cache_write_tokens": 0 }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "OpenHands" }, "google/gemini-3.5-flash-lite": { "accuracy": 37.161, "latency": 501.522, "stderr": 4.627, "cost_per_test": 0.833111, "token_totals": { "input_tokens": 387765124, "output_tokens": 3360015, "reasoning_tokens": 658865, "cache_read_tokens": 307681641, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "xiaomi/mimo-v2.5-pro": { "accuracy": 34.113, "latency": 2403.764, "stderr": 4.529, "cost_per_test": 0.166747, "token_totals": { "input_tokens": 7242687, "output_tokens": 3078284, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": "OpenHands" }, "google/gemini-3.1-pro-preview": { "accuracy": 32.034, "latency": 1211.906, "stderr": 4.342, "cost_per_test": 3.825202, "token_totals": { "input_tokens": 722761650, "output_tokens": 1941198, "reasoning_tokens": 260725, "cache_read_tokens": 678233584, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "zai/glm-5.3-flash": { "accuracy": 30.759, "latency": 5027.281, "stderr": 5.277, "cost_per_test": 2.348805, "token_totals": { "input_tokens": 7308851703, "output_tokens": 11916569, "reasoning_tokens": null, "cache_read_tokens": 7228379520, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": "OpenHands" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 26.097, "latency": 3276.224, "stderr": 5.075, "cost_per_test": 1.277298, "token_totals": { "input_tokens": 2524890518, "output_tokens": 10662405, "reasoning_tokens": 4838232, "cache_read_tokens": 2460443284, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "OpenHands" }, "alibaba/qwen3.6-plus": { "accuracy": 25.565, "latency": 2289.977, "stderr": 3.906, "cost_per_test": 5.447636, "token_totals": { "input_tokens": 479493282, "output_tokens": 2653745, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": "OpenHands" }, "google/gemini-3-flash-preview": { "accuracy": 20.204, "latency": 806.66, "stderr": 3.946, "cost_per_test": 0.942067, "token_totals": { "input_tokens": 317204132, "output_tokens": 2183311, "reasoning_tokens": 357653, "cache_read_tokens": 279403697, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "grok/grok-4.3": { "accuracy": 19.403, "latency": 589.407, "stderr": 4.243, "cost_per_test": 1.28472, "token_totals": { "input_tokens": 240295477, "output_tokens": 2046068, "reasoning_tokens": 220495, "cache_read_tokens": 235309534, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "thinkingmachines/inkling": { "accuracy": 19.214, "latency": 1105.492, "stderr": 3.659, "cost_per_test": 1.605949, "token_totals": { "input_tokens": 385384338, "output_tokens": 2166011, "reasoning_tokens": 0, "cache_read_tokens": 380152294, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": "OpenHands" }, "thinkingmachines/inkling-small": { "accuracy": 19.052, "latency": 2949.469, "stderr": 3.403, "cost_per_test": 0.432264, "token_totals": { "input_tokens": 294407624, "output_tokens": 2135670, "reasoning_tokens": null, "cache_read_tokens": 289327616, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": "OpenHands" }, "kimi/kimi-k2.5-thinking": { "accuracy": 17.536, "latency": 2570.381, "stderr": 3.258, "cost_per_test": 0.879292, "token_totals": { "input_tokens": 289848467, "output_tokens": 2490783, "reasoning_tokens": 123400, "cache_read_tokens": 283745267, "cache_write_tokens": 0 }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": "OpenHands" }, "minimax/MiniMax-M2.7": { "accuracy": 11.926, "latency": 1377.326, "stderr": 2.857, "cost_per_test": 2.819109, "token_totals": { "input_tokens": 470065443, "output_tokens": 3235220, "reasoning_tokens": 171505, "cache_read_tokens": 63061859, "cache_write_tokens": 325666 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": "OpenHands" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 11.393, "latency": 775.882, "stderr": 3.129, "cost_per_test": 1.306474, "token_totals": { "input_tokens": 1024173457, "output_tokens": 4124891, "reasoning_tokens": 878548, "cache_read_tokens": 1007363608, "cache_write_tokens": 16704885 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "OpenHands" }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 7.636, "latency": 2217.462, "stderr": 2.783, "cost_per_test": null, "token_totals": { "input_tokens": 1965727676, "output_tokens": 5440855, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": "OpenHands" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 4.063, "latency": 301.651, "stderr": 2.058, "cost_per_test": 0.771885, "token_totals": { "input_tokens": 7403442417, "output_tokens": 1887400, "reasoning_tokens": 76700, "cache_read_tokens": 7061849983, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": "OpenHands" }, "inception/mercury-2.5": { "accuracy": 3.683, "latency": 193.041, "stderr": 1.407, "cost_per_test": 0.378016, "token_totals": { "input_tokens": 198612247, "output_tokens": 1426011, "reasoning_tokens": 1298893, "cache_read_tokens": 121617600, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": "OpenHands" }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 3.4, "latency": 1451.032, "stderr": 1.291, "cost_per_test": 0.233022, "token_totals": { "input_tokens": 666603077, "output_tokens": 7351064, "reasoning_tokens": null, "cache_read_tokens": 618088896, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": "OpenHands" }, "ant/ling-3.0-flash-2607": { "accuracy": 2.914, "latency": 1005.598, "stderr": 2.072, "cost_per_test": 0.292199, "token_totals": { "input_tokens": 803530511, "output_tokens": 3846224, "reasoning_tokens": null, "cache_read_tokens": 768376791, "cache_write_tokens": 0 }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": "OpenHands" }, "mistralai/mistral-medium-3.5": { "accuracy": 2.888, "latency": 2470.533, "stderr": 1.713, "cost_per_test": 16.735503, "token_totals": { "input_tokens": 538194513, "output_tokens": 3931115, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": "OpenHands" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 572.711, "stderr": 0.0, "cost_per_test": 0.51, "token_totals": { "input_tokens": 53939702, "output_tokens": 1042512, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": "OpenHands" }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 959.662, "stderr": 0.0, "cost_per_test": 0.128799, "token_totals": { "input_tokens": 535193452, "output_tokens": 3346611, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": "OpenHands" } }, "finance_agent": { "google/gemini-3.8-flash": { "accuracy": 61.435, "latency": 200.952, "stderr": 0.128, "cost_per_test": 2.000457, "token_totals": { "input_tokens": 1118010382, "output_tokens": 25909358, "reasoning_tokens": 18104089, "cache_read_tokens": 719355707, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 60.599, "latency": 304.205, "stderr": 0.279, "cost_per_test": 0.76972, "token_totals": { "input_tokens": 424942458, "output_tokens": 12218400, "reasoning_tokens": 6559929, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 59.958, "latency": 198.929, "stderr": 2.06, "cost_per_test": 0.756512, "token_totals": { "input_tokens": 508981355, "output_tokens": 16233804, "reasoning_tokens": 9783963, "cache_read_tokens": 331627422, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 59.042, "latency": 181.334, "stderr": 0.272, "cost_per_test": 1.477884, "token_totals": { "input_tokens": 711083728, "output_tokens": 16887053, "reasoning_tokens": 10058516, "cache_read_tokens": 391282104, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 58.901, "latency": 341.936, "stderr": 0.444, "cost_per_test": 0.741132, "token_totals": { "input_tokens": 413247670, "output_tokens": 14548497, "reasoning_tokens": 8843980, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 58.877, "latency": 1094.165, "stderr": 2.058, "cost_per_test": 8.346696, "token_totals": { "input_tokens": 458174812, "output_tokens": 33030168, "reasoning_tokens": 19377315, "cache_read_tokens": 228829928, "cache_write_tokens": 229316981 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 58.633, "latency": 598.424, "stderr": 0.077, "cost_per_test": 5.124473, "token_totals": { "input_tokens": 543793980, "output_tokens": 22921224, "reasoning_tokens": 9150666, "cache_read_tokens": 253099566, "cache_write_tokens": 290670709 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 58.587, "latency": 1987.81, "stderr": 0.17, "cost_per_test": 9.215734, "token_totals": { "input_tokens": 763751733, "output_tokens": 109304455, "reasoning_tokens": 94811735, "cache_read_tokens": 413590027, "cache_write_tokens": 350117722 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 58.103, "latency": 1903.336, "stderr": 0.673, "cost_per_test": 6.552431, "token_totals": { "input_tokens": 1121526513, "output_tokens": 137870326, "reasoning_tokens": 120529788, "cache_read_tokens": 555448124, "cache_write_tokens": 566027579 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 57.861, "latency": 322.279, "stderr": 0.231, "cost_per_test": 2.505488, "token_totals": { "input_tokens": 1227406624, "output_tokens": 26782944, "reasoning_tokens": 20834822, "cache_read_tokens": 707613227, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 57.85, "latency": 890.519, "stderr": 2.061, "cost_per_test": 0.047625, "token_totals": { "input_tokens": 333663746, "output_tokens": 26908803, "reasoning_tokens": null, "cache_read_tokens": 172014080, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 57.339, "latency": 648.075, "stderr": 0.574, "cost_per_test": 0.198909, "token_totals": { "input_tokens": 305876104, "output_tokens": 13378019, "reasoning_tokens": 7659784, "cache_read_tokens": 127922645, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 57.207, "latency": 414.77, "stderr": 0.789, "cost_per_test": 0.723391, "token_totals": { "input_tokens": 363982665, "output_tokens": 15279252, "reasoning_tokens": 10574096, "cache_read_tokens": 177594427, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 56.314, "latency": 610.952, "stderr": 0.84, "cost_per_test": 8.061563, "token_totals": { "input_tokens": 311882539, "output_tokens": 19969523, "reasoning_tokens": null, "cache_read_tokens": 110357130, "cache_write_tokens": 201446408 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 56.296, "latency": 224.899, "stderr": 0.183, "cost_per_test": 1.402901, "token_totals": { "input_tokens": 627928399, "output_tokens": 16297009, "reasoning_tokens": 11292206, "cache_read_tokens": 320603458, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 56.277, "latency": 421.99, "stderr": 0.462, "cost_per_test": 0.071536, "token_totals": { "input_tokens": 349885254, "output_tokens": 15078803, "reasoning_tokens": 7475980, "cache_read_tokens": 154212352, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 55.84, "latency": 949.539, "stderr": 2.066, "cost_per_test": 1.072768, "token_totals": { "input_tokens": 363886559, "output_tokens": 36729670, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 55.063, "latency": 1424.632, "stderr": 0.308, "cost_per_test": 0.5903, "token_totals": { "input_tokens": 316560636, "output_tokens": 41494337, "reasoning_tokens": 34844249, "cache_read_tokens": 129727253, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 55.044, "latency": 772.153, "stderr": 0.309, "cost_per_test": 0.279224, "token_totals": { "input_tokens": 533549683, "output_tokens": 52574268, "reasoning_tokens": 45760878, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 54.927, "latency": 375.066, "stderr": 0.679, "cost_per_test": 0.044792, "token_totals": { "input_tokens": 401804903, "output_tokens": 19410118, "reasoning_tokens": null, "cache_read_tokens": 155120896, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 54.436, "latency": 1449.852, "stderr": 2.071, "cost_per_test": 3.629197, "token_totals": { "input_tokens": 651703808, "output_tokens": 67770045, "reasoning_tokens": 58991995, "cache_read_tokens": 334920434, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 54.36, "latency": 876.783, "stderr": 0.531, "cost_per_test": 1.066994, "token_totals": { "input_tokens": 422084036, "output_tokens": 16313955, "reasoning_tokens": 10785593, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 53.918, "latency": 547.397, "stderr": 0.159, "cost_per_test": 4.218538, "token_totals": { "input_tokens": 309993285, "output_tokens": 17301258, "reasoning_tokens": null, "cache_read_tokens": 82013466, "cache_write_tokens": 227923817 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 53.909, "latency": 792.086, "stderr": 0.518, "cost_per_test": 0.750759, "token_totals": { "input_tokens": 164610121, "output_tokens": 10102005, "reasoning_tokens": null, "cache_read_tokens": 75956629, "cache_write_tokens": 88646581 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 53.756, "latency": 1165.258, "stderr": 0.846, "cost_per_test": 1.253419, "token_totals": { "input_tokens": 322058918, "output_tokens": 6216567, "reasoning_tokens": 4178719, "cache_read_tokens": 276042752, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 53.682, "latency": 1000.937, "stderr": 0.67, "cost_per_test": 1.657732, "token_totals": { "input_tokens": 495041785, "output_tokens": 24551729, "reasoning_tokens": 16279727, "cache_read_tokens": 260943147, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 53.54, "latency": 692.672, "stderr": 2.078, "cost_per_test": 6.81746, "token_totals": { "input_tokens": 240568066, "output_tokens": 22397666, "reasoning_tokens": 16813955, "cache_read_tokens": 70591920, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 53.481, "latency": 364.376, "stderr": 0.39, "cost_per_test": 0.207465, "token_totals": { "input_tokens": 354713236, "output_tokens": 33772547, "reasoning_tokens": 28648435, "cache_read_tokens": 182250763, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.7": { "accuracy": 52.251, "latency": 932.724, "stderr": 0.355, "cost_per_test": 2.719136, "token_totals": { "input_tokens": 827415903, "output_tokens": 23428536, "reasoning_tokens": 15352966, "cache_read_tokens": 444856320, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 51.76, "latency": 662.269, "stderr": 0.55, "cost_per_test": 4.149702, "token_totals": { "input_tokens": 423716492, "output_tokens": 17794792, "reasoning_tokens": 13531292, "cache_read_tokens": 217525163, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 51.509, "latency": 360.437, "stderr": 0.49, "cost_per_test": 4.032269, "token_totals": { "input_tokens": 370221415, "output_tokens": 12293417, "reasoning_tokens": null, "cache_read_tokens": 140274986, "cache_write_tokens": 229852824 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 51.035, "latency": 695.588, "stderr": 0.329, "cost_per_test": 2.409559, "token_totals": { "input_tokens": 421005921, "output_tokens": 20173634, "reasoning_tokens": null, "cache_read_tokens": 230989311, "cache_write_tokens": 189800855 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 50.593, "latency": 1174.928, "stderr": 0.578, "cost_per_test": 1.236977, "token_totals": { "input_tokens": 313950822, "output_tokens": 28653788, "reasoning_tokens": 23202069, "cache_read_tokens": 139669632, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 50.393, "latency": 974.518, "stderr": 0.249, "cost_per_test": 0.880465, "token_totals": { "input_tokens": 419761736, "output_tokens": 30621918, "reasoning_tokens": 25089786, "cache_read_tokens": 218761173, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 49.873, "latency": 897.881, "stderr": 0.232, "cost_per_test": 0.124338, "token_totals": { "input_tokens": 413604271, "output_tokens": 45731435, "reasoning_tokens": 41794427, "cache_read_tokens": 199389488, "cache_write_tokens": 214114651 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 49.699, "latency": 473.158, "stderr": 0.878, "cost_per_test": 0.713981, "token_totals": { "input_tokens": 271639159, "output_tokens": 12183429, "reasoning_tokens": 7820405, "cache_read_tokens": 98781099, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 49.517, "latency": 490.245, "stderr": 0.465, "cost_per_test": 0.276154, "token_totals": { "input_tokens": 406112640, "output_tokens": 27423276, "reasoning_tokens": 22726163, "cache_read_tokens": 213584768, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-sol": { "accuracy": 49.05, "latency": 539.907, "stderr": 0.58, "cost_per_test": 2.122121, "token_totals": { "input_tokens": 416329027, "output_tokens": 22746494, "reasoning_tokens": 18774690, "cache_read_tokens": 155710451, "cache_write_tokens": 260528040 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 48.553, "latency": 621.733, "stderr": 2.096, "cost_per_test": 0.748532, "token_totals": { "input_tokens": 422396163, "output_tokens": 41880461, "reasoning_tokens": 34693358, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 48.349, "latency": 289.014, "stderr": 0.458, "cost_per_test": 1.156495, "token_totals": { "input_tokens": 346429154, "output_tokens": 13008523, "reasoning_tokens": 7208719, "cache_read_tokens": 193601579, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 48.269, "latency": 496.894, "stderr": 0.437, "cost_per_test": 0.32345, "token_totals": { "input_tokens": 302569216, "output_tokens": 10069655, "reasoning_tokens": null, "cache_read_tokens": 126041678, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 47.776, "latency": 635.178, "stderr": 0.727, "cost_per_test": 0.553214, "token_totals": { "input_tokens": 104973358, "output_tokens": 3989677, "reasoning_tokens": 2814264, "cache_read_tokens": 19536384, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 47.442, "latency": 142.913, "stderr": 0.513, "cost_per_test": 0.389575, "token_totals": { "input_tokens": 670526775, "output_tokens": 10900400, "reasoning_tokens": 7520808, "cache_read_tokens": 196667284, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling": { "accuracy": 46.597, "latency": 1082.125, "stderr": 0.925, "cost_per_test": 0.996264, "token_totals": { "input_tokens": 882963514, "output_tokens": 24911364, "reasoning_tokens": null, "cache_read_tokens": 647224491, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 45.36, "latency": 2153.503, "stderr": 0.447, "cost_per_test": 1.201815, "token_totals": { "input_tokens": 609985753, "output_tokens": 72941360, "reasoning_tokens": 70078573, "cache_read_tokens": 362827776, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 44.899, "latency": 1629.639, "stderr": 0.733, "cost_per_test": 1.141587, "token_totals": { "input_tokens": 846905588, "output_tokens": 23540852, "reasoning_tokens": 0, "cache_read_tokens": 488314911, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 44.083, "latency": 1526.183, "stderr": 0.653, "cost_per_test": 0.879086, "token_totals": { "input_tokens": 508078472, "output_tokens": 22058372, "reasoning_tokens": 17233163, "cache_read_tokens": 284032811, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 42.982, "latency": 208.258, "stderr": 1.206, "cost_per_test": 1.591906, "token_totals": { "input_tokens": 365193664, "output_tokens": 7730374, "reasoning_tokens": 5787767, "cache_read_tokens": 123797501, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 42.551, "latency": 246.543, "stderr": 0.434, "cost_per_test": 0.570657, "token_totals": { "input_tokens": 657633680, "output_tokens": 15218094, "reasoning_tokens": 12099810, "cache_read_tokens": 261501224, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 41.502, "latency": 497.373, "stderr": 1.193, "cost_per_test": 0.206445, "token_totals": { "input_tokens": 507567154, "output_tokens": 8732144, "reasoning_tokens": 5054819, "cache_read_tokens": 314066453, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 41.257, "latency": 994.255, "stderr": 0.832, "cost_per_test": 0.230271, "token_totals": { "input_tokens": 696484546, "output_tokens": 28023249, "reasoning_tokens": null, "cache_read_tokens": 580562347, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 40.846, "latency": 611.907, "stderr": 0.131, "cost_per_test": 0.814379, "token_totals": { "input_tokens": 488661479, "output_tokens": 11569301, "reasoning_tokens": 8412225, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 38.22, "latency": 542.045, "stderr": 1.04, "cost_per_test": 0.357661, "token_totals": { "input_tokens": 446705484, "output_tokens": 9996981, "reasoning_tokens": 7272392, "cache_read_tokens": 192899072, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 38.217, "latency": 334.538, "stderr": 1.185, "cost_per_test": 0.15925, "token_totals": { "input_tokens": 584481947, "output_tokens": 10638763, "reasoning_tokens": 7882802, "cache_read_tokens": 325180587, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 37.728, "latency": 615.499, "stderr": 0.373, "cost_per_test": 0.848345, "token_totals": { "input_tokens": 348106276, "output_tokens": 15533931, "reasoning_tokens": 12549303, "cache_read_tokens": 125272960, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 37.667, "latency": 253.882, "stderr": 0.149, "cost_per_test": null, "token_totals": { "input_tokens": 408960062, "output_tokens": 7705379, "reasoning_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 36.728, "latency": 339.26, "stderr": 0.109, "cost_per_test": 0.08693, "token_totals": { "input_tokens": 616264480, "output_tokens": 8440426, "reasoning_tokens": 5101469, "cache_read_tokens": 360948032, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 35.792, "latency": 751.754, "stderr": 1.989, "cost_per_test": 0.294389, "token_totals": { "input_tokens": 324471186, "output_tokens": 7830906, "reasoning_tokens": 5301673, "cache_read_tokens": 171400704, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 32.103, "latency": 768.405, "stderr": 0.967, "cost_per_test": 1.908513, "token_totals": { "input_tokens": 448123641, "output_tokens": 24450234, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 31.01, "latency": 176.769, "stderr": 0.564, "cost_per_test": 0.602129, "token_totals": { "input_tokens": 314188143, "output_tokens": 7984201, "reasoning_tokens": null, "cache_read_tokens": 140237697, "cache_write_tokens": 172251987 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 30.314, "latency": 157.02, "stderr": 0.275, "cost_per_test": 0.05047, "token_totals": { "input_tokens": 593259509, "output_tokens": 13185323, "reasoning_tokens": null, "cache_read_tokens": 426536000, "cache_write_tokens": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 29.988, "latency": 54.778, "stderr": 0.643, "cost_per_test": 0.142482, "token_totals": { "input_tokens": 309249776, "output_tokens": 4518540, "reasoning_tokens": 3033245, "cache_read_tokens": 88769598, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 28.492, "latency": 351.413, "stderr": 0.315, "cost_per_test": 0.791809, "token_totals": { "input_tokens": 181695255, "output_tokens": 12341665, "reasoning_tokens": 9833132, "cache_read_tokens": 66807493, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 27.887, "latency": 803.528, "stderr": 0.508, "cost_per_test": 0.18348, "token_totals": { "input_tokens": 573686662, "output_tokens": 15857919, "reasoning_tokens": null, "cache_read_tokens": 404654261, "cache_write_tokens": 9164196 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "inception/mercury-2.5": { "accuracy": 18.556, "latency": 95.056, "stderr": 0.255, "cost_per_test": 0.106441, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 18.5, "latency": 539.61, "stderr": 0.882, "cost_per_test": 0.028554, "token_totals": { "input_tokens": 335136335, "output_tokens": 19277054, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "terminal_bench_2_1": { "anthropic/claude-opus-5-5": { "accuracy": 87.64, "latency": 383.079, "stderr": 1.716, "cost_per_test": 0.496033, "token_totals": { "input_tokens": 22792900, "output_tokens": 1198947, "reasoning_tokens": 351772, "cache_read_tokens": 20604830, "cache_write_tokens": 2181727 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 87.266, "latency": 274.44, "stderr": 0.375, "cost_per_test": 1.339493, "token_totals": { "input_tokens": 24715799, "output_tokens": 1309769, "reasoning_tokens": 764679, "cache_read_tokens": 21492398, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 85.768, "latency": 372.819, "stderr": 1.35, "cost_per_test": 1.023319, "token_totals": { "input_tokens": 29114624, "output_tokens": 1324119, "reasoning_tokens": 721900, "cache_read_tokens": 20938069, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 85.019, "latency": 500.355, "stderr": 0.991, "cost_per_test": 3.074796, "token_totals": { "input_tokens": 21892452, "output_tokens": 965107, "reasoning_tokens": 472161, "cache_read_tokens": 19902668, "cache_write_tokens": 1985257 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 84.644, "latency": 752.932, "stderr": 0.991, "cost_per_test": 0.886936, "token_totals": { "input_tokens": 35206371, "output_tokens": 1733729, "reasoning_tokens": 1113936, "cache_read_tokens": 32076924, "cache_write_tokens": 3126721 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 83.146, "latency": 298.876, "stderr": 1.297, "cost_per_test": 0.373024, "token_totals": { "input_tokens": 53182967, "output_tokens": 1568343, "reasoning_tokens": 998593, "cache_read_tokens": 50178777, "cache_write_tokens": 2943219 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 83.146, "latency": 465.828, "stderr": 1.716, "cost_per_test": 0.625494, "token_totals": { "input_tokens": 47223436, "output_tokens": 3585507, "reasoning_tokens": 379696, "cache_read_tokens": 43257785, "cache_write_tokens": 3960025 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 81.273, "latency": 554.797, "stderr": 0.375, "cost_per_test": 1.544972, "token_totals": { "input_tokens": 325795812, "output_tokens": 4741834, "reasoning_tokens": 2933371, "cache_read_tokens": 286485175, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 80.899, "latency": 608.575, "stderr": 0.649, "cost_per_test": 0.341034, "token_totals": { "input_tokens": 20494904, "output_tokens": 1244660, "reasoning_tokens": 843947, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 80.524, "latency": 504.537, "stderr": 1.35, "cost_per_test": 1.429025, "token_totals": { "input_tokens": 21092499, "output_tokens": 1498447, "reasoning_tokens": 0, "cache_read_tokens": 18355304, "cache_write_tokens": 2613456 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 79.026, "latency": 355.751, "stderr": 0.991, "cost_per_test": 0.053767, "token_totals": { "input_tokens": 67560156, "output_tokens": 2052883, "reasoning_tokens": 1203057, "cache_read_tokens": 62168050, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 79.026, "latency": 731.78, "stderr": 0.991, "cost_per_test": 0.59671, "token_totals": { "input_tokens": 198539978, "output_tokens": 3621191, "reasoning_tokens": 2589082, "cache_read_tokens": 191325278, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 78.277, "latency": 642.908, "stderr": 2.085, "cost_per_test": 0.453642, "token_totals": { "input_tokens": 37989503, "output_tokens": 2301698, "reasoning_tokens": 1606577, "cache_read_tokens": 32943360, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 77.528, "latency": 492.202, "stderr": 0.649, "cost_per_test": 1.022219, "token_totals": { "input_tokens": 195502339, "output_tokens": 3101625, "reasoning_tokens": 1363982, "cache_read_tokens": 167065334, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 77.528, "latency": 519.739, "stderr": 2.247, "cost_per_test": 0.472552, "token_totals": { "input_tokens": 42753712, "output_tokens": 2043151, "reasoning_tokens": 1345891, "cache_read_tokens": 37760047, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 76.404, "latency": 427.412, "stderr": 0.649, "cost_per_test": 0.741883, "token_totals": { "input_tokens": 16386293, "output_tokens": 1376238, "reasoning_tokens": 851034, "cache_read_tokens": 12709120, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 76.404, "latency": 809.733, "stderr": 1.716, "cost_per_test": 0.02332, "token_totals": { "input_tokens": 77103048, "output_tokens": 3904513, "reasoning_tokens": 3173128, "cache_read_tokens": 71517419, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 74.532, "latency": 604.506, "stderr": 1.633, "cost_per_test": 0.098877, "token_totals": { "input_tokens": 56873906, "output_tokens": 6392867, "reasoning_tokens": 5656496, "cache_read_tokens": 54195840, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 74.532, "latency": 635.043, "stderr": 2.085, "cost_per_test": 0.534595, "token_totals": { "input_tokens": 74219799, "output_tokens": 2321306, "reasoning_tokens": 0, "cache_read_tokens": 70230688, "cache_write_tokens": 3987009 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 74.157, "latency": 373.223, "stderr": 1.124, "cost_per_test": 0.868179, "token_totals": { "input_tokens": 135043681, "output_tokens": 2323941, "reasoning_tokens": 196148, "cache_read_tokens": 108305986, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 73.783, "latency": 552.153, "stderr": 1.633, "cost_per_test": 1.174404, "token_totals": { "input_tokens": 224181261, "output_tokens": 3381533, "reasoning_tokens": 1807265, "cache_read_tokens": 190452944, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 73.408, "latency": 797.952, "stderr": 1.498, "cost_per_test": 1.076601, "token_totals": { "input_tokens": 98794136, "output_tokens": 2621813, "reasoning_tokens": 1801692, "cache_read_tokens": 78833280, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 73.034, "latency": 425.094, "stderr": 1.716, "cost_per_test": 0.029222, "token_totals": { "input_tokens": 68771981, "output_tokens": 2862029, "reasoning_tokens": 2313628, "cache_read_tokens": 64567254, "cache_write_tokens": 4144652 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 72.285, "latency": 857.636, "stderr": 0.375, "cost_per_test": 0.663249, "token_totals": { "input_tokens": 153551412, "output_tokens": 3070064, "reasoning_tokens": 2095052, "cache_read_tokens": 132688964, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 71.91, "latency": 929.902, "stderr": 0.649, "cost_per_test": 2.409772, "token_totals": { "input_tokens": 84371822, "output_tokens": 5125339, "reasoning_tokens": 0, "cache_read_tokens": 76658876, "cache_write_tokens": 7553615 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 71.536, "latency": 786.604, "stderr": 1.982, "cost_per_test": 0.30672, "token_totals": { "input_tokens": 46365317, "output_tokens": 2447978, "reasoning_tokens": 1787588, "cache_read_tokens": 42442496, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 70.787, "latency": 410.249, "stderr": 1.124, "cost_per_test": 0.582677, "token_totals": { "input_tokens": 33939400, "output_tokens": 2667201, "reasoning_tokens": 2048217, "cache_read_tokens": 26933702, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 69.663, "latency": 879.273, "stderr": 0.649, "cost_per_test": 0.499472, "token_totals": { "input_tokens": 131957427, "output_tokens": 3332616, "reasoning_tokens": 2186108, "cache_read_tokens": 122415847, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 69.288, "latency": 624.772, "stderr": 0.991, "cost_per_test": 0.417396, "token_totals": { "input_tokens": 138955979, "output_tokens": 3124998, "reasoning_tokens": 1933296, "cache_read_tokens": 136207264, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 68.539, "latency": 816.587, "stderr": 1.297, "cost_per_test": 1.977087, "token_totals": { "input_tokens": 109753697, "output_tokens": 3484336, "reasoning_tokens": 0, "cache_read_tokens": 103807239, "cache_write_tokens": 5773113 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 67.79, "latency": 624.415, "stderr": 4.416, "cost_per_test": 0.283943, "token_totals": { "input_tokens": 26039357, "output_tokens": 1388617, "reasoning_tokens": 651797, "cache_read_tokens": 23426304, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.2": { "accuracy": 67.79, "latency": 820.506, "stderr": 0.991, "cost_per_test": 0.425184, "token_totals": { "input_tokens": 58211167, "output_tokens": 3186166, "reasoning_tokens": 2550689, "cache_read_tokens": 50590720, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 48000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 67.79, "latency": 939.123, "stderr": 1.873, "cost_per_test": 0.072656, "token_totals": { "input_tokens": 62497955, "output_tokens": 4342791, "reasoning_tokens": 3692566, "cache_read_tokens": 56788224, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 67.416, "latency": 890.917, "stderr": 0.649, "cost_per_test": 0.448084, "token_totals": { "input_tokens": 43506113, "output_tokens": 2658371, "reasoning_tokens": 2141241, "cache_read_tokens": 36047403, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 67.041, "latency": 683.03, "stderr": 1.35, "cost_per_test": 0.021068, "token_totals": { "input_tokens": 54370624, "output_tokens": 4785981, "reasoning_tokens": 3945320, "cache_read_tokens": 51580757, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 62.921, "latency": 965.737, "stderr": 2.247, "cost_per_test": 0.028403, "token_totals": { "input_tokens": 87337009, "output_tokens": 3334054, "reasoning_tokens": 0, "cache_read_tokens": 80932267, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 61.049, "latency": 935.196, "stderr": 0.375, "cost_per_test": 0.33233, "token_totals": { "input_tokens": 20525184, "output_tokens": 1673824, "reasoning_tokens": 1260207, "cache_read_tokens": 15239680, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 60.674, "latency": 775.239, "stderr": 5.207, "cost_per_test": 0.021658, "token_totals": { "input_tokens": 86304814, "output_tokens": 3550628, "reasoning_tokens": 2582392, "cache_read_tokens": 81263296, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 58.427, "latency": 892.585, "stderr": 5.254, "cost_per_test": 0.677989, "token_totals": { "input_tokens": 95606478, "output_tokens": 4179262, "reasoning_tokens": 3317358, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 57.303, "latency": 704.225, "stderr": 0.649, "cost_per_test": 0.572392, "token_totals": { "input_tokens": 52390316, "output_tokens": 1600704, "reasoning_tokens": 0, "cache_read_tokens": 49128739, "cache_write_tokens": 3211949 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 57.303, "latency": 866.253, "stderr": 5.273, "cost_per_test": 0.043613, "token_totals": { "input_tokens": 55124187, "output_tokens": 2331687, "reasoning_tokens": 1538667, "cache_read_tokens": 51288960, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 55.056, "latency": 603.689, "stderr": 2.339, "cost_per_test": 0.127848, "token_totals": { "input_tokens": 112525802, "output_tokens": 2949484, "reasoning_tokens": 0, "cache_read_tokens": 107994240, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "tencent/hy4-preview": { "accuracy": 55.056, "latency": 1109.129, "stderr": 3.433, "cost_per_test": 0.140288, "token_totals": { "input_tokens": 58874316, "output_tokens": 2524449, "reasoning_tokens": 1360245, "cache_read_tokens": 54651691, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 54.682, "latency": 673.036, "stderr": 4.556, "cost_per_test": 0.286739, "token_totals": { "input_tokens": 20716925, "output_tokens": 4555260, "reasoning_tokens": 3735861, "cache_read_tokens": 15580075, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 54.682, "latency": 862.374, "stderr": 1.498, "cost_per_test": 0.199883, "token_totals": { "input_tokens": 41799035, "output_tokens": 3293030, "reasoning_tokens": 2648603, "cache_read_tokens": 39518421, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 53.933, "latency": 376.158, "stderr": 1.297, "cost_per_test": 0.154312, "token_totals": { "input_tokens": 40724231, "output_tokens": 2528658, "reasoning_tokens": 1486431, "cache_read_tokens": 31587323, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 53.558, "latency": 932.358, "stderr": 2.085, "cost_per_test": 0.138449, "token_totals": { "input_tokens": 21929461, "output_tokens": 1568451, "reasoning_tokens": 0, "cache_read_tokens": 18714965, "cache_write_tokens": 0 }, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 53.558, "latency": 1023.416, "stderr": 0.749, "cost_per_test": 0.205973, "token_totals": { "input_tokens": 64485322, "output_tokens": 2119689, "reasoning_tokens": 0, "cache_read_tokens": 53014239, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 53.184, "latency": 778.109, "stderr": 1.633, "cost_per_test": 0.201902, "token_totals": { "input_tokens": 22196424, "output_tokens": 2290370, "reasoning_tokens": 1654262, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 52.809, "latency": 654.983, "stderr": 0.649, "cost_per_test": 0.095849, "token_totals": { "input_tokens": 53627914, "output_tokens": 1867100, "reasoning_tokens": 1116365, "cache_read_tokens": 44188800, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 50.936, "latency": 584.116, "stderr": 2.925, "cost_per_test": null, "token_totals": { "input_tokens": 111289804, "output_tokens": 2704899, "reasoning_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 50.187, "latency": 416.383, "stderr": 0.991, "cost_per_test": 0.337052, "token_totals": { "input_tokens": 247145811, "output_tokens": 2141665, "reasoning_tokens": 1125438, "cache_read_tokens": 183334301, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 50.187, "latency": 593.826, "stderr": 2.456, "cost_per_test": 0.067399, "token_totals": { "input_tokens": 210224480, "output_tokens": 3744590, "reasoning_tokens": 0, "cache_read_tokens": 194621013, "cache_write_tokens": 0 }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 50.187, "latency": 833.528, "stderr": 1.35, "cost_per_test": 0.043393, "token_totals": { "input_tokens": 136334405, "output_tokens": 7151114, "reasoning_tokens": 0, "cache_read_tokens": 118584064, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 50.187, "latency": 1078.158, "stderr": 1.498, "cost_per_test": 0.239191, "token_totals": { "input_tokens": 36128386, "output_tokens": 3648724, "reasoning_tokens": 2947326, "cache_read_tokens": 34026923, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 48.689, "latency": 812.638, "stderr": 3.066, "cost_per_test": 0.066012, "token_totals": { "input_tokens": 90437019, "output_tokens": 1897074, "reasoning_tokens": 0, "cache_read_tokens": 87265378, "cache_write_tokens": 388109 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 47.566, "latency": 771.551, "stderr": 0.375, "cost_per_test": 0.630691, "token_totals": { "input_tokens": 226227937, "output_tokens": 3179128, "reasoning_tokens": 0, "cache_read_tokens": 220634923, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 44.195, "latency": 305.726, "stderr": 0.991, "cost_per_test": 0.407362, "token_totals": { "input_tokens": 28275678, "output_tokens": 2268536, "reasoning_tokens": 1952049, "cache_read_tokens": 18837397, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 43.82, "latency": 649.489, "stderr": 1.946, "cost_per_test": 0.363822, "token_totals": { "input_tokens": 84235092, "output_tokens": 4076150, "reasoning_tokens": 0, "cache_read_tokens": 80999717, "cache_write_tokens": 2656167 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 41.948, "latency": 470.713, "stderr": 2.622, "cost_per_test": 0.982203, "token_totals": { "input_tokens": 232260469, "output_tokens": 2597032, "reasoning_tokens": 1420634, "cache_read_tokens": 220796992, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 41.948, "latency": 1051.079, "stderr": 0.749, "cost_per_test": 0.085844, "token_totals": { "input_tokens": 34513305, "output_tokens": 1049061, "reasoning_tokens": 355219, "cache_read_tokens": 32430174, "cache_write_tokens": 0 }, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 41.573, "latency": 757.485, "stderr": 2.973, "cost_per_test": 0.072321, "token_totals": { "input_tokens": 30849406, "output_tokens": 3569635, "reasoning_tokens": 2418488, "cache_read_tokens": 23307435, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 38.951, "latency": 989.169, "stderr": 2.701, "cost_per_test": 1.190984, "token_totals": { "input_tokens": 55841015, "output_tokens": 2964803, "reasoning_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 34.082, "latency": 278.342, "stderr": 0.991, "cost_per_test": 0.084894, "token_totals": { "input_tokens": 36882464, "output_tokens": 2980609, "reasoning_tokens": 2461322, "cache_read_tokens": 27270935, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 34.082, "latency": 678.28, "stderr": 0.991, "cost_per_test": 0.2108, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 10.861, "latency": 1028.327, "stderr": 2.622, "cost_per_test": 0.014296, "token_totals": { "input_tokens": 73183797, "output_tokens": 2194003, "reasoning_tokens": 0, "cache_read_tokens": 70641781, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "emb": { "anthropic/claude-fable-5-1": { "accuracy": 76.67, "latency": 1771.558, "stderr": 2.085, "cost_per_test": 15.93249, "token_totals": { "input_tokens": 454757375, "output_tokens": 12499169, "reasoning_tokens": 8150917, "cache_read_tokens": 378586242, "cache_write_tokens": 76164267 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 75.938, "latency": 2106.434, "stderr": 2.383, "cost_per_test": 11.018974, "token_totals": { "input_tokens": 922243024, "output_tokens": 23704546, "reasoning_tokens": 18569025, "cache_read_tokens": 861165257, "cache_write_tokens": 61067293 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 75.713, "latency": 2204.991, "stderr": 2.437, "cost_per_test": 8.792863, "token_totals": { "input_tokens": 2367951142, "output_tokens": 29328538, "reasoning_tokens": 22320203, "cache_read_tokens": 2311427347, "cache_write_tokens": 56507215 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 73.666, "latency": 1594.153, "stderr": 2.468, "cost_per_test": 12.354419, "token_totals": { "input_tokens": 345067688, "output_tokens": 11953058, "reasoning_tokens": null, "cache_read_tokens": 318538087, "cache_write_tokens": 26523742 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 73.556, "latency": 1558.673, "stderr": 2.24, "cost_per_test": 6.071566, "token_totals": { "input_tokens": 686017503, "output_tokens": 12323715, "reasoning_tokens": 7957163, "cache_read_tokens": 656949908, "cache_write_tokens": 29058427 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 72.339, "latency": 590.258, "stderr": 2.373, "cost_per_test": 6.010773, "token_totals": { "input_tokens": 318866006, "output_tokens": 5457571, "reasoning_tokens": 3316634, "cache_read_tokens": 276762112, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 72.196, "latency": 769.197, "stderr": 2.42, "cost_per_test": 8.243862, "token_totals": { "input_tokens": 2960728904, "output_tokens": 21722815, "reasoning_tokens": 13162823, "cache_read_tokens": 2787511478, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 71.7, "latency": 804.28, "stderr": 2.475, "cost_per_test": 5.827063, "token_totals": { "input_tokens": 123989946, "output_tokens": 5542631, "reasoning_tokens": 3051513, "cache_read_tokens": 104449111, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 71.53, "latency": 621.736, "stderr": 2.351, "cost_per_test": 1.276882, "token_totals": { "input_tokens": 216104819, "output_tokens": 5241605, "reasoning_tokens": 3222968, "cache_read_tokens": 204982294, "cache_write_tokens": 11114719 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 71.326, "latency": 650.914, "stderr": 2.262, "cost_per_test": 5.694001, "token_totals": { "input_tokens": 1686304951, "output_tokens": 18606388, "reasoning_tokens": 7782800, "cache_read_tokens": 1546827567, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 69.373, "latency": 2632.285, "stderr": 2.606, "cost_per_test": 12.05844, "token_totals": { "input_tokens": 833619980, "output_tokens": 22136588, "reasoning_tokens": null, "cache_read_tokens": 790543547, "cache_write_tokens": 43067000 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 68.515, "latency": 1041.052, "stderr": 2.809, "cost_per_test": 0.141708, "token_totals": { "input_tokens": 362285754, "output_tokens": 11007243, "reasoning_tokens": 8590626, "cache_read_tokens": 332678280, "cache_write_tokens": 29598418 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 67.426, "latency": 829.829, "stderr": 3.056, "cost_per_test": 2.546868, "token_totals": { "input_tokens": 1115802604, "output_tokens": 13342216, "reasoning_tokens": 8360823, "cache_read_tokens": 1083342803, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 67.116, "latency": 660.047, "stderr": 2.936, "cost_per_test": 0.366502, "token_totals": { "input_tokens": 653086981, "output_tokens": 6955414, "reasoning_tokens": 4412805, "cache_read_tokens": 604381577, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 66.991, "latency": 2097.023, "stderr": 3.045, "cost_per_test": 6.481284, "token_totals": { "input_tokens": 531961928, "output_tokens": 15529729, "reasoning_tokens": 10549516, "cache_read_tokens": 448911360, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 66.399, "latency": 2093.262, "stderr": 3.045, "cost_per_test": 3.480845, "token_totals": { "input_tokens": 404448426, "output_tokens": 9088004, "reasoning_tokens": 6399483, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 66.316, "latency": 2985.704, "stderr": 3.006, "cost_per_test": 10.293794, "token_totals": { "input_tokens": 3099396585, "output_tokens": 25999381, "reasoning_tokens": null, "cache_read_tokens": 3034388542, "cache_write_tokens": 64983558 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 66.205, "latency": 813.024, "stderr": 3.11, "cost_per_test": 2.193037, "token_totals": { "input_tokens": 314065634, "output_tokens": 7024530, "reasoning_tokens": 4854711, "cache_read_tokens": 286789081, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 65.459, "latency": 1521.906, "stderr": 2.709, "cost_per_test": 0.108634, "token_totals": { "input_tokens": 891374421, "output_tokens": 16056047, "reasoning_tokens": 12655956, "cache_read_tokens": 861570496, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 65.411, "latency": 726.48, "stderr": 2.646, "cost_per_test": 3.893069, "token_totals": { "input_tokens": 1058720610, "output_tokens": 15239728, "reasoning_tokens": 6777930, "cache_read_tokens": 972645922, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 64.543, "latency": 910.575, "stderr": 2.869, "cost_per_test": 3.273352, "token_totals": { "input_tokens": 183023742, "output_tokens": 5181007, "reasoning_tokens": 3094376, "cache_read_tokens": 167235072, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 63.728, "latency": 2315.595, "stderr": 3.106, "cost_per_test": 12.756352, "token_totals": { "input_tokens": 1214546122, "output_tokens": 18458044, "reasoning_tokens": 10154511, "cache_read_tokens": 1174122388, "cache_write_tokens": 40413529 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 63.548, "latency": 1123.555, "stderr": 2.752, "cost_per_test": 5.173115, "token_totals": { "input_tokens": 1617865425, "output_tokens": 16984726, "reasoning_tokens": 10956816, "cache_read_tokens": 1520002205, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 62.858, "latency": 3313.924, "stderr": 3.141, "cost_per_test": 0.328999, "token_totals": { "input_tokens": 847310790, "output_tokens": 17554862, "reasoning_tokens": 14594001, "cache_read_tokens": 813521152, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.6": { "accuracy": 62.727, "latency": 3178.533, "stderr": 3.085, "cost_per_test": 3.056382, "token_totals": { "input_tokens": 292867165, "output_tokens": 18238122, "reasoning_tokens": 12738109, "cache_read_tokens": 269312000, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 62.712, "latency": 1374.999, "stderr": 2.956, "cost_per_test": 3.552349, "token_totals": { "input_tokens": 1152349029, "output_tokens": 12397789, "reasoning_tokens": 7178696, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.2": { "accuracy": 61.532, "latency": 3532.407, "stderr": 2.879, "cost_per_test": 4.465213, "token_totals": { "input_tokens": 1022219898, "output_tokens": 19460303, "reasoning_tokens": 14618756, "cache_read_tokens": 930948672, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 60.154, "latency": 3255.853, "stderr": 3.276, "cost_per_test": 7.22787, "token_totals": { "input_tokens": 898138518, "output_tokens": 21113414, "reasoning_tokens": null, "cache_read_tokens": 854339701, "cache_write_tokens": 43790957 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 60.066, "latency": 3467.209, "stderr": 3.352, "cost_per_test": 2.655924, "token_totals": { "input_tokens": 599112401, "output_tokens": 12051178, "reasoning_tokens": 8874470, "cache_read_tokens": 572733440, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 59.661, "latency": 1370.794, "stderr": 3.029, "cost_per_test": 4.556437, "token_totals": { "input_tokens": 821326544, "output_tokens": 18031108, "reasoning_tokens": 13367587, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "tencent/hy4-preview": { "accuracy": 58.076, "latency": 1839.41, "stderr": 3.136, "cost_per_test": 0.787396, "token_totals": { "input_tokens": 824410363, "output_tokens": 10431429, "reasoning_tokens": 7444891, "cache_read_tokens": 799662720, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 57.855, "latency": 2387.544, "stderr": 2.739, "cost_per_test": 2.195977, "token_totals": { "input_tokens": 861752101, "output_tokens": 14639278, "reasoning_tokens": 0, "cache_read_tokens": 826875904, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 57.208, "latency": 780.458, "stderr": 3.227, "cost_per_test": 0.234152, "token_totals": { "input_tokens": 893420650, "output_tokens": 13279281, "reasoning_tokens": 10184478, "cache_read_tokens": 884618493, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 57.041, "latency": 1957.165, "stderr": 2.853, "cost_per_test": 2.857267, "token_totals": { "input_tokens": 275851986, "output_tokens": 9259862, "reasoning_tokens": 4794212, "cache_read_tokens": 207838976, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 56.978, "latency": 1027.476, "stderr": 3.127, "cost_per_test": 2.313636, "token_totals": { "input_tokens": 829404547, "output_tokens": 10741088, "reasoning_tokens": 5180580, "cache_read_tokens": 769467634, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 56.977, "latency": 1574.097, "stderr": 3.391, "cost_per_test": 0.383709, "token_totals": { "input_tokens": 1018385162, "output_tokens": 15135599, "reasoning_tokens": 11153857, "cache_read_tokens": 1008679680, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 56.361, "latency": 891.512, "stderr": 3.153, "cost_per_test": 1.112498, "token_totals": { "input_tokens": 368051977, "output_tokens": 9069516, "reasoning_tokens": 4627614, "cache_read_tokens": 350123312, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 56.344, "latency": 2172.231, "stderr": 3.319, "cost_per_test": 3.787827, "token_totals": { "input_tokens": 942177449, "output_tokens": 18177772, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 55.932, "latency": 4495.202, "stderr": 3.561, "cost_per_test": 0.713033, "token_totals": { "input_tokens": 3932720612, "output_tokens": 29249606, "reasoning_tokens": null, "cache_read_tokens": 3849386240, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 55.228, "latency": 1747.98, "stderr": 3.112, "cost_per_test": 0.221264, "token_totals": { "input_tokens": 455429765, "output_tokens": 11928848, "reasoning_tokens": 6975136, "cache_read_tokens": 430971584, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 55.092, "latency": 1837.949, "stderr": 3.127, "cost_per_test": 0.083304, "token_totals": { "input_tokens": 599516539, "output_tokens": 12515553, "reasoning_tokens": 6917639, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.5": { "accuracy": 52.911, "latency": 1107.838, "stderr": 3.111, "cost_per_test": 1.55492, "token_totals": { "input_tokens": 150960387, "output_tokens": 7436273, "reasoning_tokens": 2117149, "cache_read_tokens": 129340672, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 52.803, "latency": 2247.236, "stderr": 3.059, "cost_per_test": 1.08141, "token_totals": { "input_tokens": 864870937, "output_tokens": 15254147, "reasoning_tokens": 11469965, "cache_read_tokens": 855589504, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 52.616, "latency": 708.582, "stderr": 2.974, "cost_per_test": 3.769246, "token_totals": { "input_tokens": 703037870, "output_tokens": 5442617, "reasoning_tokens": 2522069, "cache_read_tokens": 656983985, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 51.623, "latency": 822.144, "stderr": 3.053, "cost_per_test": 0.796757, "token_totals": { "input_tokens": 541981728, "output_tokens": 9803133, "reasoning_tokens": 3184172, "cache_read_tokens": 527403648, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 49.345, "latency": 2209.243, "stderr": 3.181, "cost_per_test": 0.597724, "token_totals": { "input_tokens": 414159309, "output_tokens": 10957606, "reasoning_tokens": 5555265, "cache_read_tokens": 355632384, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 47.759, "latency": 1890.343, "stderr": 2.946, "cost_per_test": 2.091762, "token_totals": { "input_tokens": 1398807070, "output_tokens": 13708644, "reasoning_tokens": null, "cache_read_tokens": 1372552733, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 45.426, "latency": 2601.215, "stderr": 3.645, "cost_per_test": 2.146984, "token_totals": { "input_tokens": 688505179, "output_tokens": 26360621, "reasoning_tokens": 22515833, "cache_read_tokens": 616310016, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 44.751, "latency": 1767.043, "stderr": 3.121, "cost_per_test": 0.57312, "token_totals": { "input_tokens": 1149607508, "output_tokens": 16413434, "reasoning_tokens": 10666857, "cache_read_tokens": 1066555648, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 43.231, "latency": 493.632, "stderr": 3.105, "cost_per_test": 1.231783, "token_totals": { "input_tokens": 1498941214, "output_tokens": 10717779, "reasoning_tokens": 4101020, "cache_read_tokens": 1299388807, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling": { "accuracy": 38.362, "latency": 1581.835, "stderr": 3.273, "cost_per_test": 1.671783, "token_totals": { "input_tokens": 658436029, "output_tokens": 9281665, "reasoning_tokens": null, "cache_read_tokens": 641195264, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 36.832, "latency": 2207.467, "stderr": 3.325, "cost_per_test": 0.179543, "token_totals": { "input_tokens": 952446737, "output_tokens": 16573128, "reasoning_tokens": null, "cache_read_tokens": 886140288, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 32.867, "latency": 1873.595, "stderr": 2.814, "cost_per_test": 2.866775, "token_totals": { "input_tokens": 521107748, "output_tokens": 8556725, "reasoning_tokens": 1171101, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 31.762, "latency": 1173.625, "stderr": 2.956, "cost_per_test": 0.249301, "token_totals": { "input_tokens": 254752442, "output_tokens": 6362193, "reasoning_tokens": null, "cache_read_tokens": 245337472, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 31.735, "latency": 2382.355, "stderr": 2.805, "cost_per_test": null, "token_totals": { "input_tokens": 591493477, "output_tokens": 12813242, "reasoning_tokens": null, "cache_read_tokens": 153934848, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 28.469, "latency": 3796.855, "stderr": 2.23, "cost_per_test": 0.786639, "token_totals": { "input_tokens": 515628666, "output_tokens": 6819932, "reasoning_tokens": 689099, "cache_read_tokens": 500772956, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 28.451, "latency": 1207.259, "stderr": 2.431, "cost_per_test": 1.658686, "token_totals": { "input_tokens": 547048102, "output_tokens": 8578062, "reasoning_tokens": 1143349, "cache_read_tokens": 26409876, "cache_write_tokens": 3330859 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 26.017, "latency": 328.69, "stderr": 2.912, "cost_per_test": 0.453183, "token_totals": { "input_tokens": 252451404, "output_tokens": 4842419, "reasoning_tokens": 1535428, "cache_read_tokens": 210062842, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 25.006, "latency": 735.485, "stderr": 2.462, "cost_per_test": 0.179925, "token_totals": { "input_tokens": 486672516, "output_tokens": 18585242, "reasoning_tokens": null, "cache_read_tokens": 379610210, "cache_write_tokens": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 22.664, "latency": 718.845, "stderr": 2.392, "cost_per_test": 1.030784, "token_totals": { "input_tokens": 426603491, "output_tokens": 8226592, "reasoning_tokens": 1108260, "cache_read_tokens": 408024356, "cache_write_tokens": 18501707 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 18.394, "latency": 271.547, "stderr": 1.667, "cost_per_test": 0.634124, "token_totals": { "input_tokens": 139153568, "output_tokens": 4590085, "reasoning_tokens": 340343, "cache_read_tokens": 118693351, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 16.987, "latency": 2359.322, "stderr": 1.452, "cost_per_test": 0.182153, "token_totals": { "input_tokens": 1042500185, "output_tokens": 30098913, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 11.739, "latency": 367.615, "stderr": 1.553, "cost_per_test": 0.469926, "token_totals": { "input_tokens": 82567089, "output_tokens": 1859488, "reasoning_tokens": 251920, "cache_read_tokens": 71571428, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 11.185, "latency": 3082.124, "stderr": 1.637, "cost_per_test": 10.603072, "token_totals": { "input_tokens": 428119963, "output_tokens": 6269299, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 8.823, "latency": 301.681, "stderr": 1.271, "cost_per_test": 0.419811, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 8.629, "latency": 67.007, "stderr": 1.088, "cost_per_test": 0.069424, "token_totals": { "input_tokens": 77708032, "output_tokens": 1187254, "reasoning_tokens": 588837, "cache_read_tokens": 62784855, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "code_migration": { "anthropic/claude-sonnet-5-5": { "accuracy": 70.131, "latency": 12151.11, "stderr": 4.777, "cost_per_test": 66.963572, "token_totals": { "input_tokens": 18672812024, "output_tokens": 47538207, "reasoning_tokens": null, "cache_read_tokens": 18443798779, "cache_write_tokens": 228955399 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 68.901, "latency": 10312.167, "stderr": 4.723, "cost_per_test": 104.165309, "token_totals": { "input_tokens": 14447217682, "output_tokens": 38623427, "reasoning_tokens": null, "cache_read_tokens": 14262241543, "cache_write_tokens": 184928335 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 68.462, "latency": 3477.726, "stderr": 4.581, "cost_per_test": 44.392054, "token_totals": { "input_tokens": 1029187893, "output_tokens": 9499581, "reasoning_tokens": 3212601, "cache_read_tokens": 977435512, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 60.033, "latency": 15033.475, "stderr": 5.359, "cost_per_test": 72.424868, "token_totals": { "input_tokens": 10264913505, "output_tokens": 29788772, "reasoning_tokens": null, "cache_read_tokens": 8854118675, "cache_write_tokens": 1410755640 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 56.92, "latency": 5089.673, "stderr": 4.599, "cost_per_test": 14.564951, "token_totals": { "input_tokens": 2322801542, "output_tokens": 10239389, "reasoning_tokens": 3831388, "cache_read_tokens": 2297501469, "cache_write_tokens": 25272650 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 56.68, "latency": 10031.127, "stderr": 5.055, "cost_per_test": 55.28024, "token_totals": { "input_tokens": 5714388105, "output_tokens": 20649057, "reasoning_tokens": null, "cache_read_tokens": 5642498395, "cache_write_tokens": 71858364 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 54.635, "latency": 6526.389, "stderr": 5.222, "cost_per_test": 108.499119, "token_totals": { "input_tokens": 5597593771, "output_tokens": 20933498, "reasoning_tokens": null, "cache_read_tokens": 5561285040, "cache_write_tokens": 36280807 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 52.397, "latency": 3444.398, "stderr": 4.956, "cost_per_test": 25.76956, "token_totals": { "input_tokens": 1727817656, "output_tokens": 7915451, "reasoning_tokens": 2058586, "cache_read_tokens": 1641043968, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 48.344, "latency": 4682.008, "stderr": 4.769, "cost_per_test": 14.559091, "token_totals": { "input_tokens": 5439866954, "output_tokens": 17951662, "reasoning_tokens": 7222274, "cache_read_tokens": 5385918220, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 48.248, "latency": 2978.471, "stderr": 4.816, "cost_per_test": 10.959637, "token_totals": { "input_tokens": 1603103372, "output_tokens": 7887791, "reasoning_tokens": 2595938, "cache_read_tokens": 1542349503, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 46.942, "latency": 4454.295, "stderr": 4.833, "cost_per_test": 30.057323, "token_totals": { "input_tokens": 2581769862, "output_tokens": 18356907, "reasoning_tokens": null, "cache_read_tokens": 2555676483, "cache_write_tokens": 26072355 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 46.854, "latency": 6779.828, "stderr": 4.852, "cost_per_test": 0.753694, "token_totals": { "input_tokens": 4064264660, "output_tokens": 15996575, "reasoning_tokens": 9301483, "cache_read_tokens": 4049037820, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 46.812, "latency": 7628.207, "stderr": 4.75, "cost_per_test": 3.436542, "token_totals": { "input_tokens": 3662384405, "output_tokens": 15049371, "reasoning_tokens": 8167976, "cache_read_tokens": 3623399040, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Openrouter", "harness": null }, "openai/gpt-5.5": { "accuracy": 45.528, "latency": 1847.974, "stderr": 4.67, "cost_per_test": 6.16951, "token_totals": { "input_tokens": 421702201, "output_tokens": 3776161, "reasoning_tokens": 1102490, "cache_read_tokens": 407000576, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 44.774, "latency": 3404.333, "stderr": 4.762, "cost_per_test": 2.797004, "token_totals": { "input_tokens": 3661685805, "output_tokens": 9104000, "reasoning_tokens": 3615927, "cache_read_tokens": 3481296159, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 44.603, "latency": 5425.271, "stderr": 4.957, "cost_per_test": 14.765608, "token_totals": { "input_tokens": 1617793063, "output_tokens": 14398758, "reasoning_tokens": 7964660, "cache_read_tokens": 1574035200, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 44.377, "latency": 2950.117, "stderr": 5.004, "cost_per_test": 0.547293, "token_totals": { "input_tokens": 1720559222, "output_tokens": 11387800, "reasoning_tokens": 6330094, "cache_read_tokens": 1701613975, "cache_write_tokens": 18922708 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 44.089, "latency": 6929.224, "stderr": 4.98, "cost_per_test": 33.545739, "token_totals": { "input_tokens": 9033717591, "output_tokens": 25196607, "reasoning_tokens": null, "cache_read_tokens": 8982964678, "cache_write_tokens": 50709475 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 43.919, "latency": 4311.51, "stderr": 4.867, "cost_per_test": 35.283936, "token_totals": { "input_tokens": 3488746200, "output_tokens": 13324518, "reasoning_tokens": null, "cache_read_tokens": 3466885116, "cache_write_tokens": 21843915 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 43.198, "latency": 14498.978, "stderr": 4.813, "cost_per_test": 22.959059, "token_totals": { "input_tokens": 4596186809, "output_tokens": 16492838, "reasoning_tokens": null, "cache_read_tokens": 4417489768, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.7": { "accuracy": 43.159, "latency": 4057.679, "stderr": 4.598, "cost_per_test": 35.977302, "token_totals": { "input_tokens": 2200666548, "output_tokens": 10780365, "reasoning_tokens": 4987474, "cache_read_tokens": 2075628160, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 42.329, "latency": 7395.352, "stderr": 4.792, "cost_per_test": 3.957576, "token_totals": { "input_tokens": 7771918239, "output_tokens": 19223061, "reasoning_tokens": 9157654, "cache_read_tokens": 7476027520, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 42.31, "latency": 10065.11, "stderr": 4.85, "cost_per_test": 0.682037, "token_totals": { "input_tokens": 3362298201, "output_tokens": 19744960, "reasoning_tokens": 12471457, "cache_read_tokens": 3329895936, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 42.192, "latency": 4061.108, "stderr": 4.561, "cost_per_test": 9.853488, "token_totals": { "input_tokens": 2834033709, "output_tokens": 12223208, "reasoning_tokens": 4663206, "cache_read_tokens": 2691783871, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 41.545, "latency": 12582.223, "stderr": 4.905, "cost_per_test": 19.204104, "token_totals": { "input_tokens": 9922831954, "output_tokens": 21697307, "reasoning_tokens": 9297600, "cache_read_tokens": 9342697216, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 40.368, "latency": 11194.48, "stderr": 4.998, "cost_per_test": 0.45437, "token_totals": { "input_tokens": 5518086732, "output_tokens": 25383280, "reasoning_tokens": 14174151, "cache_read_tokens": 5470014528, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 40.12, "latency": 6259.957, "stderr": 4.758, "cost_per_test": 37.369817, "token_totals": { "input_tokens": 6910501121, "output_tokens": 18164246, "reasoning_tokens": null, "cache_read_tokens": 6873237768, "cache_write_tokens": 37239446 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 38.759, "latency": 6842.515, "stderr": 4.591, "cost_per_test": 12.994404, "token_totals": { "input_tokens": 2647627607, "output_tokens": 13475023, "reasoning_tokens": 8023976, "cache_read_tokens": 2574753536, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.5": { "accuracy": 38.405, "latency": 1444.03, "stderr": 4.717, "cost_per_test": 3.485568, "token_totals": { "input_tokens": 434596005, "output_tokens": 6923870, "reasoning_tokens": 1464472, "cache_read_tokens": 419658752, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 31.558, "latency": 14719.553, "stderr": 4.478, "cost_per_test": 13.802526, "token_totals": { "input_tokens": 3345153832, "output_tokens": 10663834, "reasoning_tokens": 5329305, "cache_read_tokens": 3120951990, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 31.18, "latency": 5116.624, "stderr": 4.528, "cost_per_test": 9.331931, "token_totals": { "input_tokens": 2447457615, "output_tokens": 11812699, "reasoning_tokens": 4035584, "cache_read_tokens": 2341008584, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 30.679, "latency": 2194.294, "stderr": 4.472, "cost_per_test": 4.399308, "token_totals": { "input_tokens": 1580862286, "output_tokens": 8174928, "reasoning_tokens": 3815129, "cache_read_tokens": 1571278608, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 29.403, "latency": 1895.227, "stderr": 4.378, "cost_per_test": 3.925026, "token_totals": { "input_tokens": 1195020481, "output_tokens": 8130941, "reasoning_tokens": 3618470, "cache_read_tokens": 1165941081, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 28.997, "latency": 14661.487, "stderr": 4.688, "cost_per_test": 12.847904, "token_totals": { "input_tokens": 3179208698, "output_tokens": 13304977, "reasoning_tokens": 4012904, "cache_read_tokens": 2988589895, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 27.346, "latency": 5572.514, "stderr": 4.522, "cost_per_test": 5.208866, "token_totals": { "input_tokens": 1739210102, "output_tokens": 8052218, "reasoning_tokens": 0, "cache_read_tokens": 1711306426, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 27.148, "latency": 1162.652, "stderr": 4.407, "cost_per_test": 5.896207, "token_totals": { "input_tokens": 1268932455, "output_tokens": 8297778, "reasoning_tokens": 4261992, "cache_read_tokens": 1193161490, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 26.212, "latency": 1719.889, "stderr": 4.335, "cost_per_test": 1.04578, "token_totals": { "input_tokens": 770033456, "output_tokens": 5867778, "reasoning_tokens": 1599375, "cache_read_tokens": 765168768, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 23.163, "latency": 15822.169, "stderr": 4.493, "cost_per_test": 10.545353, "token_totals": { "input_tokens": 2163200307, "output_tokens": 12849749, "reasoning_tokens": 6884264, "cache_read_tokens": 2139278080, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 20.879, "latency": 4154.455, "stderr": 4.24, "cost_per_test": 6.593315, "token_totals": { "input_tokens": 3094038387, "output_tokens": 8632886, "reasoning_tokens": null, "cache_read_tokens": 3053969418, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 20.402, "latency": 2976.567, "stderr": 4.365, "cost_per_test": 0.237176, "token_totals": { "input_tokens": 1078559679, "output_tokens": 6929600, "reasoning_tokens": 2495015, "cache_read_tokens": 1067160640, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 20.185, "latency": 12284.669, "stderr": 4.375, "cost_per_test": 3.119218, "token_totals": { "input_tokens": 12253222324, "output_tokens": 16502163, "reasoning_tokens": null, "cache_read_tokens": 12135768320, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 17.911, "latency": 703.322, "stderr": 4.278, "cost_per_test": 1.592808, "token_totals": { "input_tokens": 190208614, "output_tokens": 2033653, "reasoning_tokens": 949508, "cache_read_tokens": 172927611, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 16.042, "latency": 15445.227, "stderr": 4.134, "cost_per_test": 13.951333, "token_totals": { "input_tokens": 2053108137, "output_tokens": 11346877, "reasoning_tokens": 5449513, "cache_read_tokens": 2030185983, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 14.304, "latency": 2156.542, "stderr": 3.516, "cost_per_test": 0.414091, "token_totals": { "input_tokens": 277149402, "output_tokens": 3679131, "reasoning_tokens": 1037798, "cache_read_tokens": 255845120, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 14.15, "latency": 1581.968, "stderr": 4.028, "cost_per_test": 0.380246, "token_totals": { "input_tokens": 557355446, "output_tokens": 4598752, "reasoning_tokens": 3034232, "cache_read_tokens": 526517248, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 14.049, "latency": 17218.195, "stderr": 4.228, "cost_per_test": 23.72095, "token_totals": { "input_tokens": 2864467525, "output_tokens": 13875014, "reasoning_tokens": 8034654, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 13.354, "latency": 2307.263, "stderr": 3.859, "cost_per_test": 0.099161, "token_totals": { "input_tokens": 965689741, "output_tokens": 6738110, "reasoning_tokens": 2461679, "cache_read_tokens": 953927872, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 13.32, "latency": 1241.908, "stderr": 3.885, "cost_per_test": 0.612969, "token_totals": { "input_tokens": 507102342, "output_tokens": 3760880, "reasoning_tokens": null, "cache_read_tokens": 500832128, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 13.015, "latency": 1774.029, "stderr": 3.851, "cost_per_test": 1.967707, "token_totals": { "input_tokens": 825775721, "output_tokens": 8502165, "reasoning_tokens": 5791384, "cache_read_tokens": 788337152, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 12.904, "latency": 1523.301, "stderr": 3.26, "cost_per_test": 1.836528, "token_totals": { "input_tokens": 204660927, "output_tokens": 3348366, "reasoning_tokens": 780902, "cache_read_tokens": 189442176, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 11.759, "latency": 1722.389, "stderr": 3.575, "cost_per_test": 2.830671, "token_totals": { "input_tokens": 803892329, "output_tokens": 3303883, "reasoning_tokens": null, "cache_read_tokens": 794874880, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 11.649, "latency": 1720.441, "stderr": 2.193, "cost_per_test": 4.35687, "token_totals": { "input_tokens": 509275820, "output_tokens": 4288631, "reasoning_tokens": 541314, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 9.953, "latency": 1269.224, "stderr": 2.869, "cost_per_test": 0.612721, "token_totals": { "input_tokens": 646389916, "output_tokens": 4739330, "reasoning_tokens": null, "cache_read_tokens": 598140537, "cache_write_tokens": 55694 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 7.649, "latency": 576.185, "stderr": 1.56, "cost_per_test": 0.895378, "token_totals": { "input_tokens": 312716907, "output_tokens": 3065113, "reasoning_tokens": null, "cache_read_tokens": 303949299, "cache_write_tokens": 8558680 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 6.508, "latency": 289.239, "stderr": 1.714, "cost_per_test": 0.34597, "token_totals": { "input_tokens": 127993388, "output_tokens": 2099189, "reasoning_tokens": 825111, "cache_read_tokens": 109412799, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 6.247, "latency": 3903.677, "stderr": 1.898, "cost_per_test": 0.954404, "token_totals": { "input_tokens": 433792371, "output_tokens": 3654987, "reasoning_tokens": 326116, "cache_read_tokens": 426714103, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.3": { "accuracy": 5.923, "latency": 168.105, "stderr": 1.617, "cost_per_test": 0.390549, "token_totals": { "input_tokens": 53768126, "output_tokens": 1038922, "reasoning_tokens": 519832, "cache_read_tokens": 45406511, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 4.936, "latency": 1381.103, "stderr": 1.682, "cost_per_test": 0.556215, "token_totals": { "input_tokens": 233014703, "output_tokens": 2477994, "reasoning_tokens": 1104610, "cache_read_tokens": 160937318, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 4.863, "latency": 1044.108, "stderr": 1.801, "cost_per_test": 0.738732, "token_totals": { "input_tokens": 427771621, "output_tokens": 2921137, "reasoning_tokens": 2787023, "cache_read_tokens": 248786079, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 4.835, "latency": 5555.013, "stderr": 1.867, "cost_per_test": null, "token_totals": { "input_tokens": 1006719477, "output_tokens": 5063968, "reasoning_tokens": null, "cache_read_tokens": 653027328, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 4.639, "latency": 83.937, "stderr": 1.573, "cost_per_test": 0.04974, "token_totals": { "input_tokens": 29793924, "output_tokens": 648732, "reasoning_tokens": 407301, "cache_read_tokens": 24326794, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 4.625, "latency": 12506.979, "stderr": 1.556, "cost_per_test": 18.251449, "token_totals": { "input_tokens": 692954021, "output_tokens": 4062604, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 3.298, "latency": 2507.828, "stderr": 1.778, "cost_per_test": 0.267696, "token_totals": { "input_tokens": 1413150483, "output_tokens": 7291844, "reasoning_tokens": null, "cache_read_tokens": 1394388075, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 2.94, "latency": 1770.047, "stderr": 1.277, "cost_per_test": 0.189242, "token_totals": { "input_tokens": 641620038, "output_tokens": 5306639, "reasoning_tokens": null, "cache_read_tokens": 582543552, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.213, "latency": 374.437, "stderr": 0.134, "cost_per_test": 0.293771, "token_totals": { "input_tokens": 37997661, "output_tokens": 346308, "reasoning_tokens": 54428, "cache_read_tokens": 33847908, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 18013.165, "stderr": 0.0, "cost_per_test": 0.05851, "token_totals": { "input_tokens": 90813833, "output_tokens": 573991, "reasoning_tokens": null, "cache_read_tokens": 60294164, "cache_write_tokens": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null } }, "legal_research": { "meta/muse_spark_1_3_max": { "accuracy": 55.288, "latency": 317.461, "stderr": 3.456, "cost_per_test": 0.588812, "token_totals": { "input_tokens": 244521876, "output_tokens": 10802084, "reasoning_tokens": 6371235, "cache_read_tokens": 208262104, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 55.288, "latency": 1860.719, "stderr": 3.456, "cost_per_test": 6.762498, "token_totals": { "input_tokens": 626960894, "output_tokens": 23871466, "reasoning_tokens": 9873124, "cache_read_tokens": 540621000, "cache_write_tokens": 86242293 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 55.288, "latency": 3449.382, "stderr": 3.456, "cost_per_test": 23.055789, "token_totals": { "input_tokens": 1096369699, "output_tokens": 51226810, "reasoning_tokens": 31819247, "cache_read_tokens": 936337703, "cache_write_tokens": 159943683 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 50.481, "latency": 6749.531, "stderr": 3.475, "cost_per_test": 25.163725, "token_totals": { "input_tokens": 1896570995, "output_tokens": 159153473, "reasoning_tokens": 139370561, "cache_read_tokens": 1615601671, "cache_write_tokens": 280867308 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 49.519, "latency": 1337.604, "stderr": 3.475, "cost_per_test": 9.791426, "token_totals": { "input_tokens": 210052964, "output_tokens": 22537892, "reasoning_tokens": 0, "cache_read_tokens": 150055095, "cache_write_tokens": 59958741 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 49.038, "latency": 3110.171, "stderr": 3.475, "cost_per_test": 2.241388, "token_totals": { "input_tokens": 458279584, "output_tokens": 55494507, "reasoning_tokens": 0, "cache_read_tokens": 368033787, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.6": { "accuracy": 48.077, "latency": 1504.077, "stderr": 3.473, "cost_per_test": 1.530204, "token_totals": { "input_tokens": 274776010, "output_tokens": 12377024, "reasoning_tokens": 7368696, "cache_read_tokens": 203687808, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 48.077, "latency": 4631.383, "stderr": 3.473, "cost_per_test": 21.605566, "token_totals": { "input_tokens": 1487117357, "output_tokens": 70127005, "reasoning_tokens": 61207392, "cache_read_tokens": 1241425920, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 48.077, "latency": 5588.989, "stderr": 3.473, "cost_per_test": 14.826571, "token_totals": { "input_tokens": 2742328281, "output_tokens": 179385399, "reasoning_tokens": 156471291, "cache_read_tokens": 2419855815, "cache_write_tokens": 322313384 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 47.596, "latency": 3677.636, "stderr": 3.471, "cost_per_test": 2.486351, "token_totals": { "input_tokens": 544286180, "output_tokens": 42522887, "reasoning_tokens": 35664649, "cache_read_tokens": 472313472, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.7": { "accuracy": 47.115, "latency": 1274.337, "stderr": 3.469, "cost_per_test": 4.920812, "token_totals": { "input_tokens": 476673717, "output_tokens": 14028919, "reasoning_tokens": 9121242, "cache_read_tokens": 350504320, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 47.115, "latency": 1816.322, "stderr": 3.469, "cost_per_test": 0.181201, "token_totals": { "input_tokens": 249781360, "output_tokens": 20945385, "reasoning_tokens": 15558422, "cache_read_tokens": 206759787, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 45.192, "latency": 3067.486, "stderr": 3.459, "cost_per_test": 0.120196, "token_totals": { "input_tokens": 636228822, "output_tokens": 43453265, "reasoning_tokens": 0, "cache_read_tokens": 559661440, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 45.192, "latency": 3839.511, "stderr": 3.459, "cost_per_test": 0.846372, "token_totals": { "input_tokens": 312749338, "output_tokens": 50001075, "reasoning_tokens": 43836379, "cache_read_tokens": 264949824, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "kimi/kimi-k3": { "accuracy": 44.231, "latency": 2146.326, "stderr": 3.452, "cost_per_test": 3.222748, "token_totals": { "input_tokens": 671469599, "output_tokens": 19172924, "reasoning_tokens": 13488619, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 43.75, "latency": 306.893, "stderr": 3.448, "cost_per_test": 0.508097, "token_totals": { "input_tokens": 163340498, "output_tokens": 5837394, "reasoning_tokens": 2676881, "cache_read_tokens": 113477050, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 43.75, "latency": 714.036, "stderr": 3.448, "cost_per_test": 2.820942, "token_totals": { "input_tokens": 136489421, "output_tokens": 12617848, "reasoning_tokens": 0, "cache_read_tokens": 98212592, "cache_write_tokens": 38178928 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 41.827, "latency": 1545.997, "stderr": 3.428, "cost_per_test": 2.719446, "token_totals": { "input_tokens": 435277342, "output_tokens": 29025915, "reasoning_tokens": 0, "cache_read_tokens": 353355756, "cache_write_tokens": 81742641 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 41.346, "latency": 701.783, "stderr": 3.423, "cost_per_test": 0.24793, "token_totals": { "input_tokens": 410443550, "output_tokens": 29229774, "reasoning_tokens": 25253265, "cache_read_tokens": 362718592, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 41.346, "latency": 4172.869, "stderr": 3.423, "cost_per_test": 7.907299, "token_totals": { "input_tokens": 491011337, "output_tokens": 25216135, "reasoning_tokens": 19985776, "cache_read_tokens": 386393600, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 40.865, "latency": 530.813, "stderr": 3.417, "cost_per_test": 0.650063, "token_totals": { "input_tokens": 196521355, "output_tokens": 9111400, "reasoning_tokens": 5382974, "cache_read_tokens": 135601907, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 40.865, "latency": 2527.177, "stderr": 3.417, "cost_per_test": 1.088017, "token_totals": { "input_tokens": 433434003, "output_tokens": 35408474, "reasoning_tokens": 30213976, "cache_read_tokens": 380911360, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 40.385, "latency": 2265.105, "stderr": 3.41, "cost_per_test": 7.402471, "token_totals": { "input_tokens": 743971258, "output_tokens": 21289365, "reasoning_tokens": 16694105, "cache_read_tokens": 647774464, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 39.423, "latency": 1491.339, "stderr": 3.397, "cost_per_test": 10.577143, "token_totals": { "input_tokens": 290661092, "output_tokens": 21448775, "reasoning_tokens": 16238030, "cache_read_tokens": 215071044, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 38.942, "latency": 324.318, "stderr": 3.389, "cost_per_test": 1.630498, "token_totals": { "input_tokens": 581263116, "output_tokens": 14420656, "reasoning_tokens": 10782340, "cache_read_tokens": 474745174, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 38.462, "latency": 511.955, "stderr": 3.381, "cost_per_test": 3.05609, "token_totals": { "input_tokens": 280128756, "output_tokens": 7275719, "reasoning_tokens": 2846795, "cache_read_tokens": 225999114, "cache_write_tokens": 53655955 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 38.462, "latency": 989.604, "stderr": 3.381, "cost_per_test": 2.302948, "token_totals": { "input_tokens": 157941924, "output_tokens": 5815818, "reasoning_tokens": 0, "cache_read_tokens": 121172556, "cache_write_tokens": 36610896 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 37.981, "latency": 312.994, "stderr": 3.373, "cost_per_test": 0.377379, "token_totals": { "input_tokens": 126240889, "output_tokens": 5901711, "reasoning_tokens": 3106623, "cache_read_tokens": 94898720, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 37.981, "latency": 919.689, "stderr": 3.373, "cost_per_test": 0.070424, "token_totals": { "input_tokens": 376039844, "output_tokens": 18415857, "reasoning_tokens": 10404557, "cache_read_tokens": 314532114, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.5": { "accuracy": 37.981, "latency": 953.767, "stderr": 3.373, "cost_per_test": 1.086814, "token_totals": { "input_tokens": 129605210, "output_tokens": 6614295, "reasoning_tokens": 3288695, "cache_read_tokens": 75025274, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 36.538, "latency": 2373.859, "stderr": 3.347, "cost_per_test": 0.845008, "token_totals": { "input_tokens": 1755111665, "output_tokens": 88330527, "reasoning_tokens": 78699683, "cache_read_tokens": 1562540626, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 36.058, "latency": 1600.721, "stderr": 3.337, "cost_per_test": 2.159719, "token_totals": { "input_tokens": 600057696, "output_tokens": 49730866, "reasoning_tokens": 42100615, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 34.615, "latency": 288.96, "stderr": 3.307, "cost_per_test": 0.760397, "token_totals": { "input_tokens": 216485075, "output_tokens": 6173336, "reasoning_tokens": 3838872, "cache_read_tokens": 157677752, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.2": { "accuracy": 31.25, "latency": 1023.468, "stderr": 3.222, "cost_per_test": 0.889209, "token_totals": { "input_tokens": 259792071, "output_tokens": 11812805, "reasoning_tokens": 8841213, "cache_read_tokens": 207179712, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 30.769, "latency": 277.753, "stderr": 3.208, "cost_per_test": 1.305281, "token_totals": { "input_tokens": 525018336, "output_tokens": 7135911, "reasoning_tokens": 5414896, "cache_read_tokens": 442747911, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 30.288, "latency": 1266.744, "stderr": 3.194, "cost_per_test": 0.275172, "token_totals": { "input_tokens": 310561183, "output_tokens": 24287114, "reasoning_tokens": 20969113, "cache_read_tokens": 261666816, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 30.288, "latency": 2547.747, "stderr": 3.194, "cost_per_test": 0.440296, "token_totals": { "input_tokens": 1154825865, "output_tokens": 60273682, "reasoning_tokens": 55133887, "cache_read_tokens": 1053025234, "cache_write_tokens": 100812066 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 29.808, "latency": 813.838, "stderr": 3.179, "cost_per_test": 0.342955, "token_totals": { "input_tokens": 273575187, "output_tokens": 6440429, "reasoning_tokens": 0, "cache_read_tokens": 219245518, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-6-sol": { "accuracy": 28.846, "latency": 1449.451, "stderr": 3.149, "cost_per_test": 4.897717, "token_totals": { "input_tokens": 709313788, "output_tokens": 26332564, "reasoning_tokens": 22035035, "cache_read_tokens": 634566483, "cache_write_tokens": 73881332 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 28.365, "latency": 869.377, "stderr": 3.133, "cost_per_test": 0.494044, "token_totals": { "input_tokens": 271552348, "output_tokens": 8856526, "reasoning_tokens": 0, "cache_read_tokens": 246578560, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 25.481, "latency": 774.823, "stderr": 3.029, "cost_per_test": 0.141548, "token_totals": { "input_tokens": 209190661, "output_tokens": 10262319, "reasoning_tokens": 0, "cache_read_tokens": 190714496, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 25.481, "latency": 808.802, "stderr": 3.029, "cost_per_test": 1.400701, "token_totals": { "input_tokens": 151578436, "output_tokens": 6466999, "reasoning_tokens": 3918467, "cache_read_tokens": 55579586, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 25.0, "latency": 168.482, "stderr": 3.01, "cost_per_test": 0.466741, "token_totals": { "input_tokens": 116699947, "output_tokens": 4157671, "reasoning_tokens": 2875086, "cache_read_tokens": 80852090, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 23.077, "latency": 1024.619, "stderr": 2.928, "cost_per_test": 0.682218, "token_totals": { "input_tokens": 341401669, "output_tokens": 11003300, "reasoning_tokens": 8033786, "cache_read_tokens": 300570368, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 20.673, "latency": 242.05, "stderr": 2.815, "cost_per_test": 1.100145, "token_totals": { "input_tokens": 354575717, "output_tokens": 2983139, "reasoning_tokens": 2206660, "cache_read_tokens": 321431187, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 18.269, "latency": 187.7, "stderr": 2.686, "cost_per_test": 0.135395, "token_totals": { "input_tokens": 80643637, "output_tokens": 3487795, "reasoning_tokens": 2539699, "cache_read_tokens": 50273506, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 16.346, "latency": 831.995, "stderr": 2.57, "cost_per_test": 0.295949, "token_totals": { "input_tokens": 182291959, "output_tokens": 7407756, "reasoning_tokens": 4986270, "cache_read_tokens": 118507648, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 15.865, "latency": 416.437, "stderr": 2.539, "cost_per_test": 0.082648, "token_totals": { "input_tokens": 179797926, "output_tokens": 3117976, "reasoning_tokens": 0, "cache_read_tokens": 147737472, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 15.865, "latency": 1332.514, "stderr": 2.539, "cost_per_test": 0.23035, "token_totals": { "input_tokens": 207645988, "output_tokens": 4488005, "reasoning_tokens": 2618301, "cache_read_tokens": 180277739, "cache_write_tokens": 0 }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 15.865, "latency": 1853.133, "stderr": 2.539, "cost_per_test": 1.520305, "token_totals": { "input_tokens": 1570147541, "output_tokens": 3643156, "reasoning_tokens": 0, "cache_read_tokens": 1492103312, "cache_write_tokens": 0 }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.3": { "accuracy": 15.385, "latency": 84.583, "stderr": 2.508, "cost_per_test": 0.203143, "token_totals": { "input_tokens": 61403015, "output_tokens": 3577558, "reasoning_tokens": 2695531, "cache_read_tokens": 42465638, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 15.385, "latency": 515.464, "stderr": 2.508, "cost_per_test": 0.028227, "token_totals": { "input_tokens": 221182940, "output_tokens": 7577859, "reasoning_tokens": 0, "cache_read_tokens": 182576832, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 15.385, "latency": 611.366, "stderr": 2.508, "cost_per_test": null, "token_totals": { "input_tokens": 181276135, "output_tokens": 3143670, "reasoning_tokens": 0, "cache_read_tokens": 49852416, "cache_write_tokens": 0 }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 14.904, "latency": 658.954, "stderr": 2.475, "cost_per_test": 0.697997, "token_totals": { "input_tokens": 228091457, "output_tokens": 5468376, "reasoning_tokens": 3418357, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 13.942, "latency": 59.197, "stderr": 2.408, "cost_per_test": 0.124723, "token_totals": { "input_tokens": 121402055, "output_tokens": 1885651, "reasoning_tokens": 1270438, "cache_read_tokens": 56268106, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 13.942, "latency": 244.487, "stderr": 2.408, "cost_per_test": 0.261499, "token_totals": { "input_tokens": 45447294, "output_tokens": 1401239, "reasoning_tokens": 694590, "cache_read_tokens": 30450141, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 12.5, "latency": 2256.339, "stderr": 2.299, "cost_per_test": 1.864673, "token_totals": { "input_tokens": 681633294, "output_tokens": 45110362, "reasoning_tokens": 43033228, "cache_read_tokens": 570762142, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 10.577, "latency": 243.38, "stderr": 2.138, "cost_per_test": 0.26891, "token_totals": { "input_tokens": 131779295, "output_tokens": 2744297, "reasoning_tokens": 1091315, "cache_read_tokens": 106353006, "cache_write_tokens": 24601089 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 10.577, "latency": 420.647, "stderr": 2.138, "cost_per_test": 0.219167, "token_totals": { "input_tokens": 143277305, "output_tokens": 5052868, "reasoning_tokens": 3440031, "cache_read_tokens": 14839186, "cache_write_tokens": 7287836 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 9.135, "latency": 391.469, "stderr": 2.002, "cost_per_test": 0.033318, "token_totals": { "input_tokens": 251871878, "output_tokens": 3303008, "reasoning_tokens": 1325334, "cache_read_tokens": 213241152, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 9.135, "latency": 613.248, "stderr": 2.002, "cost_per_test": 1.308079, "token_totals": { "input_tokens": 148782567, "output_tokens": 6520890, "reasoning_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 6.25, "latency": 526.838, "stderr": 1.682, "cost_per_test": 0.141253, "token_totals": { "input_tokens": 386062284, "output_tokens": 6536232, "reasoning_tokens": 4445328, "cache_read_tokens": 311122688, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 4.327, "latency": 45.996, "stderr": 1.414, "cost_per_test": 0.058112, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 3.365, "latency": 20.403, "stderr": 1.253, "cost_per_test": 0.013607, "token_totals": { "input_tokens": 8137148, "output_tokens": 895676, "reasoning_tokens": 458069, "cache_read_tokens": 2433959, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 2.885, "latency": 285.48, "stderr": 1.163, "cost_per_test": 0.004365, "token_totals": { "input_tokens": 24683324, "output_tokens": 1685576, "reasoning_tokens": 0, "cache_read_tokens": 16581517, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 211.833, "stderr": 0.0, "cost_per_test": 0.036023, "token_totals": { "input_tokens": 190331294, "output_tokens": 4335412, "reasoning_tokens": 0, "cache_read_tokens": 130131626, "cache_write_tokens": 0 }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null } }, "legal_agent_benchmark": { "meta/muse_spark_1_2": { "accuracy": 25.417, "latency": 1456.304, "stderr": 3.777, "cost_per_test": 2.08555, "token_totals": { "input_tokens": 608754402, "output_tokens": 10507621, "reasoning_tokens": 826686, "cache_read_tokens": 512433248, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 23.75, "latency": 875.202, "stderr": 3.712, "cost_per_test": 2.237702, "token_totals": { "input_tokens": 1051574748, "output_tokens": 12477578, "reasoning_tokens": 2210440, "cache_read_tokens": 1009238551, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 22.083, "latency": 998.589, "stderr": 3.478, "cost_per_test": 2.734654, "token_totals": { "input_tokens": 829515346, "output_tokens": 8150145, "reasoning_tokens": 1650722, "cache_read_tokens": 685738540, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 20.0, "latency": 740.851, "stderr": 3.219, "cost_per_test": 0.800257, "token_totals": { "input_tokens": 283802188, "output_tokens": 5292532, "reasoning_tokens": 497732, "cache_read_tokens": 263652672, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 15.833, "latency": 2705.055, "stderr": 2.645, "cost_per_test": 4.008154, "token_totals": { "input_tokens": 655024488, "output_tokens": 16284504, "reasoning_tokens": 3704862, "cache_read_tokens": 617851648, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.5": { "accuracy": 12.917, "latency": 614.713, "stderr": 2.534, "cost_per_test": 2.01955, "token_totals": { "input_tokens": 220580311, "output_tokens": 9143344, "reasoning_tokens": 326470, "cache_read_tokens": 198038400, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.7": { "accuracy": 12.5, "latency": 2524.987, "stderr": 2.415, "cost_per_test": 11.130983, "token_totals": { "input_tokens": 1000748161, "output_tokens": 20963877, "reasoning_tokens": 8944068, "cache_read_tokens": 911133568, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 11.25, "latency": 1031.586, "stderr": 2.287, "cost_per_test": 0.085646, "token_totals": { "input_tokens": 751303984, "output_tokens": 11580281, "reasoning_tokens": 5709544, "cache_read_tokens": 715360896, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 11.25, "latency": 1118.646, "stderr": 2.415, "cost_per_test": 4.033621, "token_totals": { "input_tokens": 892576388, "output_tokens": 12582101, "reasoning_tokens": 5612426, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 11.25, "latency": 1613.04, "stderr": 2.166, "cost_per_test": 19.225253, "token_totals": { "input_tokens": 939267211, "output_tokens": 13345157, "reasoning_tokens": 0, "cache_read_tokens": 880023896, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 10.833, "latency": 1549.246, "stderr": 2.166, "cost_per_test": 0.216614, "token_totals": { "input_tokens": 632411237, "output_tokens": 11283302, "reasoning_tokens": 5261809, "cache_read_tokens": 600189312, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 10.833, "latency": 2842.762, "stderr": 2.287, "cost_per_test": 5.621474, "token_totals": { "input_tokens": 906473760, "output_tokens": 10525791, "reasoning_tokens": 4083382, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 10.417, "latency": 2952.44, "stderr": 2.287, "cost_per_test": 2.365468, "token_totals": { "input_tokens": 702241099, "output_tokens": 9014364, "reasoning_tokens": 3675937, "cache_read_tokens": 671264128, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 10.0, "latency": 1760.877, "stderr": 2.415, "cost_per_test": 3.656207, "token_totals": { "input_tokens": 1171742866, "output_tokens": 16120993, "reasoning_tokens": 5514918, "cache_read_tokens": 1066501383, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 9.583, "latency": 1447.796, "stderr": 2.148, "cost_per_test": 10.222944, "token_totals": { "input_tokens": 1143265902, "output_tokens": 13823660, "reasoning_tokens": 0, "cache_read_tokens": 1089433052, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 9.167, "latency": 1764.671, "stderr": 1.998, "cost_per_test": 0.98378, "token_totals": { "input_tokens": 1072794048, "output_tokens": 13132219, "reasoning_tokens": 4467767, "cache_read_tokens": 1022096320, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 8.75, "latency": 452.106, "stderr": 2.148, "cost_per_test": 2.556743, "token_totals": { "input_tokens": 811642014, "output_tokens": 14269397, "reasoning_tokens": 3601488, "cache_read_tokens": 753832806, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 8.333, "latency": 1011.024, "stderr": 2.184, "cost_per_test": 0.059087, "token_totals": { "input_tokens": 744084543, "output_tokens": 7335131, "reasoning_tokens": 2960038, "cache_read_tokens": 723421312, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 8.333, "latency": 2558.18, "stderr": 1.998, "cost_per_test": 4.302903, "token_totals": { "input_tokens": 1456140757, "output_tokens": 16324375, "reasoning_tokens": 0, "cache_read_tokens": 1398312225, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 7.5, "latency": 1313.338, "stderr": 2.148, "cost_per_test": 0.166206, "token_totals": { "input_tokens": 805739429, "output_tokens": 9454727, "reasoning_tokens": 4381477, "cache_read_tokens": 785343488, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 7.083, "latency": 1250.96, "stderr": 1.998, "cost_per_test": 2.057915, "token_totals": { "input_tokens": 582413825, "output_tokens": 8379723, "reasoning_tokens": 3962778, "cache_read_tokens": 530965248, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 6.667, "latency": 579.445, "stderr": 1.832, "cost_per_test": 0.173471, "token_totals": { "input_tokens": 721200533, "output_tokens": 8383826, "reasoning_tokens": 4048611, "cache_read_tokens": 699334016, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 6.667, "latency": 1483.703, "stderr": 1.998, "cost_per_test": 10.80884, "token_totals": { "input_tokens": 1150494190, "output_tokens": 13074265, "reasoning_tokens": 1317583, "cache_read_tokens": 1081804814, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 6.667, "latency": 3136.585, "stderr": 1.646, "cost_per_test": 0.570286, "token_totals": { "input_tokens": 3914223047, "output_tokens": 18369668, "reasoning_tokens": 0, "cache_read_tokens": 3828746240, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 6.667, "latency": 3336.857, "stderr": 1.646, "cost_per_test": 23.669537, "token_totals": { "input_tokens": 2757683085, "output_tokens": 27559556, "reasoning_tokens": 6673656, "cache_read_tokens": 2623329853, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 6.667, "latency": 6725.273, "stderr": 1.174, "cost_per_test": 46.211522, "token_totals": { "input_tokens": 2140493271, "output_tokens": 25042909, "reasoning_tokens": 9339413, "cache_read_tokens": 1833706187, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 5.417, "latency": 1489.116, "stderr": 1.174, "cost_per_test": 26.155899, "token_totals": { "input_tokens": 1119192487, "output_tokens": 10562148, "reasoning_tokens": 3650237, "cache_read_tokens": 1049116892, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 5.0, "latency": 1122.711, "stderr": 1.646, "cost_per_test": 3.042346, "token_totals": { "input_tokens": 387928425, "output_tokens": 6856417, "reasoning_tokens": 0, "cache_read_tokens": 345650547, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 5.0, "latency": 2331.654, "stderr": 1.646, "cost_per_test": 8.953859, "token_totals": { "input_tokens": 3340913248, "output_tokens": 20212831, "reasoning_tokens": 0, "cache_read_tokens": 3252147720, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 4.167, "latency": 1351.039, "stderr": 1.646, "cost_per_test": 1.460248, "token_totals": { "input_tokens": 1114902164, "output_tokens": 8397929, "reasoning_tokens": 0, "cache_read_tokens": 1070555232, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 3.75, "latency": 401.527, "stderr": 1.431, "cost_per_test": 0.737465, "token_totals": { "input_tokens": 321058309, "output_tokens": 4167657, "reasoning_tokens": 272693, "cache_read_tokens": 303855232, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 3.75, "latency": 733.852, "stderr": 1.174, "cost_per_test": 4.597894, "token_totals": { "input_tokens": 372470134, "output_tokens": 4184434, "reasoning_tokens": 1267952, "cache_read_tokens": 341294080, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 3.75, "latency": 2939.571, "stderr": 1.28, "cost_per_test": 21.37949, "token_totals": { "input_tokens": 2462576678, "output_tokens": 38472452, "reasoning_tokens": 24925173, "cache_read_tokens": 2286257168, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 3.333, "latency": 984.497, "stderr": 1.174, "cost_per_test": 1.895602, "token_totals": { "input_tokens": 434069662, "output_tokens": 9410323, "reasoning_tokens": 2894511, "cache_read_tokens": 366081207, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 2.917, "latency": 1051.793, "stderr": 1.174, "cost_per_test": 0.303552, "token_totals": { "input_tokens": 1460813994, "output_tokens": 12095727, "reasoning_tokens": 6619071, "cache_read_tokens": 1422964599, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 2.917, "latency": 3463.508, "stderr": 1.362, "cost_per_test": 16.612583, "token_totals": { "input_tokens": 5060567562, "output_tokens": 57267592, "reasoning_tokens": 37755971, "cache_read_tokens": 4882858962, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 2.5, "latency": 477.704, "stderr": 0.84, "cost_per_test": 2.11387, "token_totals": { "input_tokens": 599835437, "output_tokens": 8606879, "reasoning_tokens": 3771681, "cache_read_tokens": 537529325, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 2.5, "latency": 1370.186, "stderr": 0.833, "cost_per_test": 10.374554, "token_totals": { "input_tokens": 1221154582, "output_tokens": 10565145, "reasoning_tokens": 2786144, "cache_read_tokens": 1150617952, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 2.083, "latency": 544.233, "stderr": 0.833, "cost_per_test": 0.107095, "token_totals": { "input_tokens": 224714337, "output_tokens": 3198839, "reasoning_tokens": 0, "cache_read_tokens": 203250560, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling": { "accuracy": 2.083, "latency": 1103.634, "stderr": 0.84, "cost_per_test": 1.252023, "token_totals": { "input_tokens": 575866616, "output_tokens": 7829496, "reasoning_tokens": 0, "cache_read_tokens": 552512512, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 1.667, "latency": 419.283, "stderr": 0.0, "cost_per_test": 0.036252, "token_totals": { "input_tokens": 252892290, "output_tokens": 3100583, "reasoning_tokens": 145588, "cache_read_tokens": 232673984, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-sol": { "accuracy": 1.667, "latency": 820.101, "stderr": 0.823, "cost_per_test": 3.358015, "token_totals": { "input_tokens": 823114531, "output_tokens": 7639311, "reasoning_tokens": 3222223, "cache_read_tokens": 791317282, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 1.667, "latency": 1402.917, "stderr": 0.833, "cost_per_test": 0.818741, "token_totals": { "input_tokens": 373247916, "output_tokens": 3422680, "reasoning_tokens": 0, "cache_read_tokens": 341806731, "cache_write_tokens": 0 }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 1.667, "latency": 1870.976, "stderr": 0.833, "cost_per_test": 0.174542, "token_totals": { "input_tokens": 201049757, "output_tokens": 5401207, "reasoning_tokens": 0, "cache_read_tokens": 191047296, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 1.667, "latency": 1923.268, "stderr": 0.84, "cost_per_test": 1.57592, "token_totals": { "input_tokens": 249242887, "output_tokens": 2999380, "reasoning_tokens": 202288, "cache_read_tokens": 203585792, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 1.25, "latency": 161.511, "stderr": 0.855, "cost_per_test": 0.046313, "token_totals": { "input_tokens": 146486877, "output_tokens": 4726655, "reasoning_tokens": 0, "cache_read_tokens": 110128578, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 1.25, "latency": 656.364, "stderr": 0.833, "cost_per_test": 0.383981, "token_totals": { "input_tokens": 993049543, "output_tokens": 9388666, "reasoning_tokens": 2979345, "cache_read_tokens": 909992290, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 1.25, "latency": 888.428, "stderr": 0.84, "cost_per_test": 3.940956, "token_totals": { "input_tokens": 488567465, "output_tokens": 3914542, "reasoning_tokens": 154232, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.833, "latency": 360.353, "stderr": 0.833, "cost_per_test": 0.416609, "token_totals": { "input_tokens": 129616489, "output_tokens": 3945656, "reasoning_tokens": 242225, "cache_read_tokens": 114566187, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.833, "latency": 1024.892, "stderr": 0.833, "cost_per_test": 3.202399, "token_totals": { "input_tokens": 802970365, "output_tokens": 9301238, "reasoning_tokens": 2973597, "cache_read_tokens": 740704262, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 0.417, "latency": 168.727, "stderr": 0.0, "cost_per_test": 0.389144, "token_totals": { "input_tokens": 88472856, "output_tokens": 3683568, "reasoning_tokens": 464382, "cache_read_tokens": 71653723, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.417, "latency": 2034.322, "stderr": 0.0, "cost_per_test": null, "token_totals": { "input_tokens": 308111092, "output_tokens": 3786963, "reasoning_tokens": 0, "cache_read_tokens": 48837632, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.417, "latency": 3694.952, "stderr": 0.0, "cost_per_test": 6.074265, "token_totals": { "input_tokens": 439770514, "output_tokens": 5184631, "reasoning_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 60.561, "stderr": 0.0, "cost_per_test": 0.084048, "token_totals": { "input_tokens": 160066368, "output_tokens": 769494, "reasoning_tokens": 360545, "cache_read_tokens": 138155856, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 125.317, "stderr": 0.0, "cost_per_test": 0.192614, "token_totals": { "input_tokens": 159575776, "output_tokens": 1621389, "reasoning_tokens": 1524161, "cache_read_tokens": 55653173, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 202.527, "stderr": 0.0, "cost_per_test": 0.264375, "token_totals": { "input_tokens": 225950385, "output_tokens": 1424392, "reasoning_tokens": 580723, "cache_read_tokens": 190051973, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 232.904, "stderr": 0.0, "cost_per_test": 0.331805, "token_totals": { "input_tokens": 234750879, "output_tokens": 3541635, "reasoning_tokens": 1058834, "cache_read_tokens": 146158528, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 357.397, "stderr": 0.0, "cost_per_test": 0.045991, "token_totals": { "input_tokens": 212025841, "output_tokens": 4296441, "reasoning_tokens": 0, "cache_read_tokens": 166167424, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 363.015, "stderr": 0.0, "cost_per_test": 0.184287, "token_totals": { "input_tokens": 315810901, "output_tokens": 2855013, "reasoning_tokens": 1338430, "cache_read_tokens": 261179392, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 436.699, "stderr": 0.0, "cost_per_test": 1.149757, "token_totals": { "input_tokens": 230732445, "output_tokens": 1350000, "reasoning_tokens": 503310, "cache_read_tokens": 204622317, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 438.359, "stderr": 0.0, "cost_per_test": 0.584587, "token_totals": { "input_tokens": 174177409, "output_tokens": 2611543, "reasoning_tokens": 157743, "cache_read_tokens": 10278052, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 502.909, "stderr": 0.0, "cost_per_test": 0.986814, "token_totals": { "input_tokens": 248182901, "output_tokens": 1198831, "reasoning_tokens": 239427, "cache_read_tokens": 216554703, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 529.209, "stderr": 0.0, "cost_per_test": 0.234226, "token_totals": { "input_tokens": 214653131, "output_tokens": 1748791, "reasoning_tokens": 535291, "cache_read_tokens": 170857216, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 681.277, "stderr": 0.0, "cost_per_test": 1.001493, "token_totals": { "input_tokens": 440584578, "output_tokens": 6595483, "reasoning_tokens": 5276888, "cache_read_tokens": 389590016, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1508.281, "stderr": 0.0, "cost_per_test": 0.351359, "token_totals": { "input_tokens": 236174566, "output_tokens": 2527211, "reasoning_tokens": 124700, "cache_read_tokens": 219868250, "cache_write_tokens": 0 }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 1735.942, "stderr": 0.0, "cost_per_test": 0.028287, "token_totals": { "input_tokens": 216862233, "output_tokens": 2963970, "reasoning_tokens": 0, "cache_read_tokens": 201035952, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } } } }