{ "metadata": { "benchmark": "Code Migration", "slug": "code-migration", "description": "Can language models reimplement real-world programs in another language?", "benchmark_id": "code_migration", "family": "code_migration", "version": "1", "updated": "2026-10-29", "dataset_type": "private", "industry": "coding", "tasks": { "overall": "Overall", "cli": "CLI", "cobol": "COBOL", "code_quality": "Code Quality" }, "models": [ "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.7-plus", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "ant/ling-3.0-flash-2607", "ant/ling-3.0-flash-af-rc3", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "deepseek/deepseek-v4.1-flash-1017", "fireworks/ember-1", "fireworks/nemotron-lightning-3p5-30b-a3b", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-4.7", "inception/mercury-2.5", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k2.7-code", "kimi/kimi-k3", "meta/muse-spark-open", "meta/muse-wonder", "meta/muse-wonder-full", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/mistral-large-4", "mistralai/mistral-medium-3.5", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "stepfun/step-5-preview", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.1", "zai/glm-5.2", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "token_tasks_measured": 130, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 79 }, "tasks": { "overall": { "anthropic/claude-sonnet-5-5": { "accuracy": 69.825, "latency": 12004.483, "stderr": 4.262, "cost_per_test": 75.826721, "token_totals": { "input_tokens": 45638880163, "output_tokens": 111192842, "reasoning_tokens": 47097700, "cache_read_tokens": 45068987365, "cache_write_tokens": 569754636 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 529.9, "avg_query_seconds": 12.6, "median_query_seconds": 7.4, "avg_ttft_seconds": 12.2, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 68.173, "latency": 7891.461, "stderr": 4.349, "cost_per_test": 57.822126, "token_totals": { "input_tokens": 20014688279, "output_tokens": 51989513, "reasoning_tokens": 11752491, "cache_read_tokens": 18990619728, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 267.4, "avg_query_seconds": 27.3, "median_query_seconds": 14.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 67.74, "latency": 3510.003, "stderr": 4.216, "cost_per_test": 44.364548, "token_totals": { "input_tokens": 2477045520, "output_tokens": 22664152, "reasoning_tokens": 7634788, "cache_read_tokens": 2359018535, "cache_write_tokens": null }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 82.6, "avg_query_seconds": 28.8, "median_query_seconds": 16.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 66.646, "latency": 11925.532, "stderr": 4.329, "cost_per_test": 112.970791, "token_totals": { "input_tokens": 34205044527, "output_tokens": 88928708, "reasoning_tokens": 30916622, "cache_read_tokens": 33786255871, "cache_write_tokens": 418677436 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 426.5, "avg_query_seconds": 23.6, "median_query_seconds": 16.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 65.122, "latency": 5781.916, "stderr": 4.362, "cost_per_test": 6.505708, "token_totals": { "input_tokens": 2494667995, "output_tokens": 30703807, "reasoning_tokens": 14011754, "cache_read_tokens": 2435018778, "cache_write_tokens": 59527967 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 78.6, "avg_query_seconds": 63.3, "median_query_seconds": 33.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 62.552, "latency": 10267.754, "stderr": 4.354, "cost_per_test": 69.535349, "token_totals": { "input_tokens": 27156894108, "output_tokens": 53374698, "reasoning_tokens": 24799488, "cache_read_tokens": 25992647612, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 462.2, "avg_query_seconds": 12.3, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 61.033, "latency": 13956.531, "stderr": 4.419, "cost_per_test": 38.410556, "token_totals": { "input_tokens": 22775518104, "output_tokens": 49583224, "reasoning_tokens": 19349689, "cache_read_tokens": 22666804976, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 402.8, "avg_query_seconds": 24.0, "median_query_seconds": 12.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 57.473, "latency": 10268.729, "stderr": 4.371, "cost_per_test": 60.509587, "token_totals": { "input_tokens": 13948783938, "output_tokens": 48168536, "reasoning_tokens": 16736164, "cache_read_tokens": 13778689514, "cache_write_tokens": 170020888 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 282.8, "avg_query_seconds": 17.4, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 57.195, "latency": 5136.155, "stderr": 4.213, "cost_per_test": 15.676945, "token_totals": { "input_tokens": 5912921043, "output_tokens": 24484144, "reasoning_tokens": 8633960, "cache_read_tokens": 5853103211, "cache_write_tokens": 59517900 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 167.1, "avg_query_seconds": 14.7, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 55.065, "latency": 6559.388, "stderr": 4.605, "cost_per_test": 112.098109, "token_totals": { "input_tokens": 13549878096, "output_tokens": 49676333, "reasoning_tokens": null, "cache_read_tokens": 13465564599, "cache_write_tokens": 84247569 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 127, "avg_turns": 259.6, "avg_query_seconds": 21.9, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 54.607, "latency": 15610.474, "stderr": 4.812, "cost_per_test": 70.973455, "token_totals": { "input_tokens": 23288552351, "output_tokens": 68289749, "reasoning_tokens": 17433265, "cache_read_tokens": 20286721243, "cache_write_tokens": 3001742226 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 340.8, "avg_query_seconds": 25.3, "median_query_seconds": 11.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 52.916, "latency": 3551.876, "stderr": 4.353, "cost_per_test": 24.541448, "token_totals": { "input_tokens": 4568080385, "output_tokens": 18803275, "reasoning_tokens": 4682306, "cache_read_tokens": 4359125504, "cache_write_tokens": null }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 134.7, "avg_query_seconds": 12.2, "median_query_seconds": 8.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 47.804, "latency": 2914.132, "stderr": 4.28, "cost_per_test": 8.127254, "token_totals": { "input_tokens": 3758509701, "output_tokens": 17992895, "reasoning_tokens": 5555226, "cache_read_tokens": 3610776154, "cache_write_tokens": null }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 134.2, "avg_query_seconds": 11.1, "median_query_seconds": 7.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 47.434, "latency": 7994.848, "stderr": 4.272, "cost_per_test": 3.405133, "token_totals": { "input_tokens": 8513522261, "output_tokens": 34132436, "reasoning_tokens": 17977879, "cache_read_tokens": 8423328768, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 268.0, "avg_query_seconds": 19.5, "median_query_seconds": 12.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 47.414, "latency": 5054.737, "stderr": 4.265, "cost_per_test": 14.210071, "token_totals": { "input_tokens": 12492583130, "output_tokens": 40315710, "reasoning_tokens": 15750708, "cache_read_tokens": 12362640597, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 319.6, "avg_query_seconds": 9.2, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 47.25, "latency": 4498.693, "stderr": 4.176, "cost_per_test": 30.514184, "token_totals": { "input_tokens": 6098153872, "output_tokens": 40923766, "reasoning_tokens": null, "cache_read_tokens": 6039695136, "cache_write_tokens": 58410352 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 186.1, "avg_query_seconds": 21.9, "median_query_seconds": 13.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 45.623, "latency": 6325.865, "stderr": 4.285, "cost_per_test": 0.944694, "token_totals": { "input_tokens": 9823703780, "output_tokens": 36706023, "reasoning_tokens": 20727269, "cache_read_tokens": 9693379447, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 282.2, "avg_query_seconds": 6.7, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 45.161, "latency": 1894.953, "stderr": 4.164, "cost_per_test": 6.442253, "token_totals": { "input_tokens": 1045841839, "output_tokens": 8688574, "reasoning_tokens": 2356937, "cache_read_tokens": 1011919872, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 84.4, "avg_query_seconds": 16.7, "median_query_seconds": 8.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 44.816, "latency": 4141.411, "stderr": 4.215, "cost_per_test": 36.55237, "token_totals": { "input_tokens": 5272600636, "output_tokens": 24417825, "reasoning_tokens": 10925026, "cache_read_tokens": 4982596096, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 178.6, "avg_query_seconds": 16.5, "median_query_seconds": 9.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.6": { "accuracy": 44.572, "latency": 5429.239, "stderr": 4.479, "cost_per_test": 14.575491, "token_totals": { "input_tokens": 3807271564, "output_tokens": 32806961, "reasoning_tokens": 17603001, "cache_read_tokens": 3723214976, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 165.0, "avg_query_seconds": 27.6, "median_query_seconds": 13.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 44.568, "latency": 4971.306, "stderr": 4.231, "cost_per_test": 6.23191, "token_totals": { "input_tokens": 13237438344, "output_tokens": 65785285, "reasoning_tokens": 37387395, "cache_read_tokens": 13121904638, "cache_write_tokens": 115469948 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 244.1, "avg_query_seconds": 9.2, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 44.55, "latency": 3514.758, "stderr": 4.244, "cost_per_test": 1.882918, "token_totals": { "input_tokens": 9254505783, "output_tokens": 21170273, "reasoning_tokens": 8066626, "cache_read_tokens": 8792844546, "cache_write_tokens": null }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 262.0, "avg_query_seconds": 7.1, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 44.392, "latency": 7026.544, "stderr": 4.246, "cost_per_test": 35.310438, "token_totals": { "input_tokens": 21994394229, "output_tokens": 58769418, "reasoning_tokens": null, "cache_read_tokens": 21880629287, "cache_write_tokens": 113661938 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 129, "avg_turns": 399.2, "avg_query_seconds": 13.9, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 44.222, "latency": 13633.637, "stderr": 4.289, "cost_per_test": 24.909755, "token_totals": { "input_tokens": 12154169576, "output_tokens": 38328579, "reasoning_tokens": null, "cache_read_tokens": 11812129280, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 128, "avg_turns": 341.6, "avg_query_seconds": 16.8, "median_query_seconds": 8.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 44.21, "latency": 7843.507, "stderr": 4.455, "cost_per_test": 1.858025, "token_totals": { "input_tokens": 31908355564, "output_tokens": 52632882, "reasoning_tokens": 25529336, "cache_read_tokens": 31797847932, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 577.2, "avg_query_seconds": 5.2, "median_query_seconds": 3.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 43.878, "latency": 4327.01, "stderr": 4.22, "cost_per_test": 36.387681, "token_totals": { "input_tokens": 8244099899, "output_tokens": 29218633, "reasoning_tokens": null, "cache_read_tokens": 8195001963, "cache_write_tokens": 49058419 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 299.4, "avg_query_seconds": 10.5, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 43.013, "latency": 10136.401, "stderr": 4.32, "cost_per_test": 0.691641, "token_totals": { "input_tokens": 7970131550, "output_tokens": 46647096, "reasoning_tokens": 29124060, "cache_read_tokens": 7897231360, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 197.4, "avg_query_seconds": 41.2, "median_query_seconds": 23.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 42.554, "latency": 3023.191, "stderr": 4.415, "cost_per_test": 0.600876, "token_totals": { "input_tokens": 4477712731, "output_tokens": 27061727, "reasoning_tokens": 14173752, "cache_read_tokens": 4430079486, "cache_write_tokens": 47578681 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 139.9, "avg_query_seconds": 15.1, "median_query_seconds": 8.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 41.54, "latency": 12726.884, "stderr": 4.301, "cost_per_test": 18.581309, "token_totals": { "input_tokens": 23018772982, "output_tokens": 49370423, "reasoning_tokens": 20791249, "cache_read_tokens": 21697498752, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 486.3, "avg_query_seconds": 15.4, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 40.927, "latency": 11538.747, "stderr": 4.351, "cost_per_test": 0.493563, "token_totals": { "input_tokens": 14568390083, "output_tokens": 61489266, "reasoning_tokens": 34926129, "cache_read_tokens": 14452928000, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 256.3, "avg_query_seconds": 31.8, "median_query_seconds": 18.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 39.892, "latency": 5979.033, "stderr": 4.138, "cost_per_test": 38.507642, "token_totals": { "input_tokens": 16787188570, "output_tokens": 40290013, "reasoning_tokens": null, "cache_read_tokens": 16710245252, "cache_write_tokens": 76886240 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 437.3, "avg_query_seconds": 12.5, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 38.977, "latency": 12856.895, "stderr": 4.533, "cost_per_test": 7.606885, "token_totals": { "input_tokens": 18056589533, "output_tokens": 46511084, "reasoning_tokens": 0, "cache_read_tokens": 17950566400, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 420.9, "avg_query_seconds": 17.6, "median_query_seconds": 12.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 38.631, "latency": 7700.455, "stderr": 4.382, "cost_per_test": 3.605899, "token_totals": { "input_tokens": 18460496784, "output_tokens": 43837702, "reasoning_tokens": 19926321, "cache_read_tokens": 17886795648, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 444.9, "avg_query_seconds": 8.4, "median_query_seconds": 6.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 37.87, "latency": 6856.367, "stderr": 4.143, "cost_per_test": 12.635818, "token_totals": { "input_tokens": 5977854484, "output_tokens": 30070206, "reasoning_tokens": 17648559, "cache_read_tokens": 5824142016, "cache_write_tokens": null }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 241.8, "avg_query_seconds": 23.9, "median_query_seconds": 14.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.5": { "accuracy": 36.596, "latency": 1408.81, "stderr": 4.141, "cost_per_test": 3.369806, "token_totals": { "input_tokens": 996820478, "output_tokens": 15529740, "reasoning_tokens": 3026779, "cache_read_tokens": 964539008, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 73.5, "avg_query_seconds": 13.7, "median_query_seconds": 7.5, "avg_ttft_seconds": 4.9, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 36.546, "latency": 9443.316, "stderr": 4.184, "cost_per_test": 18.491664, "token_totals": { "input_tokens": 12417243563, "output_tokens": 28013213, "reasoning_tokens": 13429274, "cache_read_tokens": 11880108404, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 379.1, "avg_query_seconds": 3.2, "median_query_seconds": 2.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 34.984, "latency": 2146.253, "stderr": 4.113, "cost_per_test": 7.756561, "token_totals": { "input_tokens": 2318467130, "output_tokens": 14241439, "reasoning_tokens": 5644282, "cache_read_tokens": 2222630912, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 114.9, "avg_query_seconds": 17.8, "median_query_seconds": 9.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 34.8, "latency": 9520.131, "stderr": 4.225, "cost_per_test": 21.464006, "token_totals": { "input_tokens": 12013539711, "output_tokens": 35771097, "reasoning_tokens": 10812716, "cache_read_tokens": 11078299995, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 129, "avg_turns": 303.1, "avg_query_seconds": 5.2, "median_query_seconds": 3.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/ember-1": { "accuracy": 32.508, "latency": 14916.444, "stderr": 4.626, "cost_per_test": 51.110833, "token_totals": { "input_tokens": 19565759850, "output_tokens": 47168944, "reasoning_tokens": 18392884, "cache_read_tokens": 19075128475, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 425.5, "avg_query_seconds": 24.3, "median_query_seconds": 15.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 31.115, "latency": 2351.314, "stderr": 4.068, "cost_per_test": 4.560329, "token_totals": { "input_tokens": 3841900354, "output_tokens": 19058781, "reasoning_tokens": 8349292, "cache_read_tokens": 3820589568, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 175.8, "avg_query_seconds": 9.9, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 30.928, "latency": 5404.829, "stderr": 4.068, "cost_per_test": 8.930416, "token_totals": { "input_tokens": 5528913845, "output_tokens": 24782032, "reasoning_tokens": 8136536, "cache_read_tokens": 5298191357, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 116, "avg_turns": 219.3, "avg_query_seconds": 5.7, "median_query_seconds": 3.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 30.561, "latency": 17441.256, "stderr": 4.221, "cost_per_test": 24.522375, "token_totals": { "input_tokens": 12077872708, "output_tokens": 52885651, "reasoning_tokens": null, "cache_read_tokens": 8073165312, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 317.6, "avg_query_seconds": 46.0, "median_query_seconds": 29.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 29.954, "latency": 1866.928, "stderr": 4.023, "cost_per_test": 3.784593, "token_totals": { "input_tokens": 2599076274, "output_tokens": 17528189, "reasoning_tokens": 7525072, "cache_read_tokens": 2535660917, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 132.3, "avg_query_seconds": 7.0, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 27.768, "latency": 5398.566, "stderr": 4.144, "cost_per_test": 5.112053, "token_totals": { "input_tokens": 3928671986, "output_tokens": 18166580, "reasoning_tokens": 0, "cache_read_tokens": 3864895520, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 251.7, "avg_query_seconds": 21.8, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 27.58, "latency": 4061.108, "stderr": 4.067, "cost_per_test": 4.616854, "token_totals": { "input_tokens": 2834033709, "output_tokens": 12223208, "reasoning_tokens": 4663206, "cache_read_tokens": 2691783871, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 26.745, "latency": 1168.454, "stderr": 4.104, "cost_per_test": 5.725122, "token_totals": { "input_tokens": 2728996153, "output_tokens": 18686686, "reasoning_tokens": 9035969, "cache_read_tokens": 2564162180, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 170.7, "avg_query_seconds": 6.3, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 26.201, "latency": 1737.978, "stderr": 4.037, "cost_per_test": 1.071016, "token_totals": { "input_tokens": 1775529861, "output_tokens": 12956898, "reasoning_tokens": 3286712, "cache_read_tokens": 1765216896, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 138.3, "avg_query_seconds": 11.2, "median_query_seconds": 5.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.1": { "accuracy": 25.768, "latency": 3502.363, "stderr": 4.087, "cost_per_test": 5.701313, "token_totals": { "input_tokens": 3165609834, "output_tokens": 13659057, "reasoning_tokens": 1820533, "cache_read_tokens": 2978224512, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 252.5, "avg_query_seconds": 13.4, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 25.392, "latency": 5945.669, "stderr": 4.161, "cost_per_test": 5.040913, "token_totals": { "input_tokens": 3443313277, "output_tokens": 14145374, "reasoning_tokens": 0, "cache_read_tokens": 3416326110, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 223.9, "avg_query_seconds": 24.8, "median_query_seconds": 14.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 23.958, "latency": 16065.728, "stderr": 4.309, "cost_per_test": 10.53816, "token_totals": { "input_tokens": 4924988504, "output_tokens": 27884870, "reasoning_tokens": 13993268, "cache_read_tokens": 4871917440, "cache_write_tokens": null }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 208.3, "avg_query_seconds": 45.8, "median_query_seconds": 25.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 21.557, "latency": 2932.892, "stderr": 4.127, "cost_per_test": 0.2383, "token_totals": { "input_tokens": 2532815772, "output_tokens": 15600313, "reasoning_tokens": 5400257, "cache_read_tokens": 2507280576, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 171.6, "avg_query_seconds": 15.5, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-spark-open": { "accuracy": 21.171, "latency": 6790.697, "stderr": 3.472, "cost_per_test": 9.042387, "token_totals": { "input_tokens": 2648676598, "output_tokens": 14008508, "reasoning_tokens": 8828852, "cache_read_tokens": 1937882464, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 118.1, "avg_query_seconds": 47.3, "median_query_seconds": 16.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 20.515, "latency": 12331.871, "stderr": 4.207, "cost_per_test": 3.128822, "token_totals": { "input_tokens": 27812131417, "output_tokens": 35322408, "reasoning_tokens": null, "cache_read_tokens": 27542076800, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 129, "avg_turns": 713.6, "avg_query_seconds": 8.9, "median_query_seconds": 5.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 19.925, "latency": 4428.262, "stderr": 3.944, "cost_per_test": 7.072659, "token_totals": { "input_tokens": 7770548615, "output_tokens": 19586346, "reasoning_tokens": null, "cache_read_tokens": 7676936773, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 357.4, "avg_query_seconds": 8.8, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 17.313, "latency": 720.574, "stderr": 3.933, "cost_per_test": 1.614622, "token_totals": { "input_tokens": 408291051, "output_tokens": 4321140, "reasoning_tokens": 1978576, "cache_read_tokens": 370531958, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 81.3, "avg_query_seconds": 8.1, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 16.099, "latency": 15450.785, "stderr": 4.101, "cost_per_test": 13.871104, "token_totals": { "input_tokens": 4493367892, "output_tokens": 24228298, "reasoning_tokens": 11292718, "cache_read_tokens": 4442013693, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 201.6, "avg_query_seconds": 70.4, "median_query_seconds": 56.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 14.467, "latency": 1594.162, "stderr": 4.025, "cost_per_test": 0.416225, "token_totals": { "input_tokens": 1294210826, "output_tokens": 10679804, "reasoning_tokens": 6879683, "cache_read_tokens": 1214781440, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 121, "avg_turns": 125.9, "avg_query_seconds": 8.0, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 14.228, "latency": 2274.632, "stderr": 3.687, "cost_per_test": 0.102802, "token_totals": { "input_tokens": 2305809055, "output_tokens": 15884330, "reasoning_tokens": 5240580, "cache_read_tokens": 2278726976, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 155.0, "avg_query_seconds": 12.3, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 14.157, "latency": 17150.223, "stderr": 4.189, "cost_per_test": 21.677954, "token_totals": { "input_tokens": 5807911522, "output_tokens": 26865532, "reasoning_tokens": 15621662, "cache_read_tokens": 0, "cache_write_tokens": null }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 216.3, "avg_query_seconds": 37.0, "median_query_seconds": 12.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 13.675, "latency": 1232.702, "stderr": 3.81, "cost_per_test": 0.596991, "token_totals": { "input_tokens": 1049910656, "output_tokens": 7907813, "reasoning_tokens": null, "cache_read_tokens": 1036819200, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 159.0, "avg_query_seconds": 4.1, "median_query_seconds": 2.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 13.069, "latency": 1531.852, "stderr": 2.858, "cost_per_test": 1.932899, "token_totals": { "input_tokens": 482293989, "output_tokens": 7687787, "reasoning_tokens": 1555360, "cache_read_tokens": 447887616, "cache_write_tokens": null }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 63.8, "avg_query_seconds": 18.3, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 12.936, "latency": 1733.471, "stderr": 3.641, "cost_per_test": 1.953198, "token_totals": { "input_tokens": 1870656126, "output_tokens": 19298168, "reasoning_tokens": 12789925, "cache_read_tokens": 1782290944, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 105.8, "avg_query_seconds": 16.0, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 12.856, "latency": 2146.993, "stderr": 2.931, "cost_per_test": 0.41923, "token_totals": { "input_tokens": 597024550, "output_tokens": 8047859, "reasoning_tokens": 2030700, "cache_read_tokens": 549853440, "cache_write_tokens": null }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 129, "avg_turns": 84.3, "avg_query_seconds": 16.4, "median_query_seconds": 7.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 11.79, "latency": 1773.502, "stderr": 3.396, "cost_per_test": 3.006967, "token_totals": { "input_tokens": 1739695483, "output_tokens": 7272245, "reasoning_tokens": null, "cache_read_tokens": 1720919936, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 128, "avg_turns": 190.5, "avg_query_seconds": 5.9, "median_query_seconds": 2.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 11.102, "latency": 1789.961, "stderr": 1.312, "cost_per_test": 4.781669, "token_totals": { "input_tokens": 1265012690, "output_tokens": 9755632, "reasoning_tokens": 1170780, "cache_read_tokens": 0, "cache_write_tokens": null }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 131.3, "avg_query_seconds": 13.0, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 8.685, "latency": 1259.318, "stderr": 1.786, "cost_per_test": 0.587624, "token_totals": { "input_tokens": 1463541240, "output_tokens": 10829853, "reasoning_tokens": null, "cache_read_tokens": 1371740469, "cache_write_tokens": 139765 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 147.1, "avg_query_seconds": 6.5, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 7.551, "latency": 593.715, "stderr": 1.056, "cost_per_test": 0.981576, "token_totals": { "input_tokens": 695588933, "output_tokens": 6615966, "reasoning_tokens": null, "cache_read_tokens": 676271243, "cache_write_tokens": 18873036 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 113.1, "avg_query_seconds": 4.2, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 6.962, "latency": 3820.711, "stderr": 1.29, "cost_per_test": 0.926909, "token_totals": { "input_tokens": 901413072, "output_tokens": 8223726, "reasoning_tokens": 677797, "cache_read_tokens": 886119691, "cache_write_tokens": null }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 105.5, "avg_query_seconds": 34.5, "median_query_seconds": 21.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.3": { "accuracy": 6.795, "latency": 180.478, "stderr": 1.195, "cost_per_test": 0.416921, "token_totals": { "input_tokens": 123525559, "output_tokens": 2325853, "reasoning_tokens": 1105443, "cache_read_tokens": 105801067, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 41.8, "avg_query_seconds": 2.9, "median_query_seconds": 1.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 6.391, "latency": 284.234, "stderr": 1.093, "cost_per_test": 0.340087, "token_totals": { "input_tokens": 274721017, "output_tokens": 4397620, "reasoning_tokens": 1689679, "cache_read_tokens": 233704955, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 60.6, "avg_query_seconds": 4.0, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 6.148, "latency": 1322.302, "stderr": 1.708, "cost_per_test": 0.525186, "token_totals": { "input_tokens": 422878800, "output_tokens": 4819392, "reasoning_tokens": 2204805, "cache_read_tokens": 277179852, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 67.1, "avg_query_seconds": 2.2, "median_query_seconds": 1.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 5.127, "latency": 13055.328, "stderr": 1.369, "cost_per_test": 18.623756, "token_totals": { "input_tokens": 1504705914, "output_tokens": 8247994, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 125, "avg_turns": 196.4, "avg_query_seconds": 20.5, "median_query_seconds": 15.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 4.905, "latency": 5470.513, "stderr": 1.589, "cost_per_test": null, "token_totals": { "input_tokens": 2299384725, "output_tokens": 12006439, "reasoning_tokens": null, "cache_read_tokens": 1440832512, "cache_write_tokens": null }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 114, "avg_turns": 199.2, "avg_query_seconds": 12.0, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 4.609, "latency": 92.63, "stderr": 1.069, "cost_per_test": 0.053028, "token_totals": { "input_tokens": 73960601, "output_tokens": 1433780, "reasoning_tokens": 895097, "cache_read_tokens": 61387242, "cache_write_tokens": null }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 30.0, "avg_query_seconds": 1.9, "median_query_seconds": 1.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 4.517, "latency": 974.083, "stderr": 1.048, "cost_per_test": 0.627223, "token_totals": { "input_tokens": 959228311, "output_tokens": 6764397, "reasoning_tokens": 6460465, "cache_read_tokens": 735400505, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 130.4, "avg_query_seconds": 2.2, "median_query_seconds": 1.2, "avg_ttft_seconds": 1.7, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 2.951, "latency": 2604.747, "stderr": 0.951, "cost_per_test": 0.267017, "token_totals": { "input_tokens": 3110660796, "output_tokens": 15411080, "reasoning_tokens": null, "cache_read_tokens": 3066735770, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 233.6, "avg_query_seconds": 3.8, "median_query_seconds": 1.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 1.824, "latency": 1770.047, "stderr": 0.866, "cost_per_test": 0.103478, "token_totals": { "input_tokens": 641620038, "output_tokens": 5306639, "reasoning_tokens": null, "cache_read_tokens": 582543552, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.314, "latency": 373.673, "stderr": 0.11, "cost_per_test": 0.289486, "token_totals": { "input_tokens": 73150780, "output_tokens": 715021, "reasoning_tokens": 118722, "cache_read_tokens": 64024414, "cache_write_tokens": 0 }, "token_tasks_measured": 130, "turn_metrics": { "tasks_measured": 130, "avg_turns": 42.1, "avg_query_seconds": 6.6, "median_query_seconds": 1.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 17934.764, "stderr": 0.0, "cost_per_test": 0.060112, "token_totals": { "input_tokens": 221735196, "output_tokens": 1404520, "reasoning_tokens": null, "cache_read_tokens": 165595602, "cache_write_tokens": null }, "token_tasks_measured": 130, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null } }, "cli": { "anthropic/claude-sonnet-5-5": { "accuracy": 69.766, "latency": 13638.865, "stderr": 2.524, "cost_per_test": 94.636889, "turn_metrics": { "tasks_measured": 120, "avg_turns": 558.0, "avg_query_seconds": 12.5, "median_query_seconds": 7.3, "avg_ttft_seconds": 12.1, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 67.564, "latency": 9667.444, "stderr": 2.774, "cost_per_test": 73.570037, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 67.463, "latency": 4293.222, "stderr": 2.551, "cost_per_test": 57.311535, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 65.528, "latency": 14209.808, "stderr": 2.719, "cost_per_test": 142.978632, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 63.496, "latency": 7039.569, "stderr": 2.811, "cost_per_test": 8.28157, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 60.087, "latency": 7275.185, "stderr": 2.841, "cost_per_test": 139.102342, "turn_metrics": { "tasks_measured": 120, "avg_turns": 267.4, "avg_query_seconds": 21.7, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 60.07, "latency": 12990.853, "stderr": 2.789, "cost_per_test": 90.462172, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 58.044, "latency": 17190.365, "stderr": 2.964, "cost_per_test": 49.149508, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 53.297, "latency": 12019.037, "stderr": 2.837, "cost_per_test": 74.846365, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 52.927, "latency": 6436.976, "stderr": 2.372, "cost_per_test": 20.385725, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 52.809, "latency": 16546.06, "stderr": 3.396, "cost_per_test": 87.094073, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 47.221, "latency": 4369.049, "stderr": 2.786, "cost_per_test": 31.186416, "turn_metrics": { "tasks_measured": 120, "avg_turns": 143.1, "avg_query_seconds": 12.2, "median_query_seconds": 8.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 40.405, "latency": 3460.538, "stderr": 2.576, "cost_per_test": 10.093854, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 40.362, "latency": 6033.476, "stderr": 2.691, "cost_per_test": 17.997369, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 40.143, "latency": 4819.54, "stderr": 2.431, "cost_per_test": 35.418678, "turn_metrics": { "tasks_measured": 120, "avg_turns": 193.2, "avg_query_seconds": 21.5, "median_query_seconds": 13.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 39.911, "latency": 9341.195, "stderr": 2.552, "cost_per_test": 4.20179, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "grok/grok-4.6": { "accuracy": 39.429, "latency": 6039.778, "stderr": 2.456, "cost_per_test": 18.261586, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 37.497, "latency": 7282.206, "stderr": 2.592, "cost_per_test": 1.150189, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 37.215, "latency": 3676.982, "stderr": 2.491, "cost_per_test": 0.773618, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 36.882, "latency": 1746.371, "stderr": 2.212, "cost_per_test": 7.539299, "turn_metrics": { "tasks_measured": 120, "avg_turns": 87.4, "avg_query_seconds": 16.7, "median_query_seconds": 8.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 36.422, "latency": 4816.662, "stderr": 2.378, "cost_per_test": 46.752741, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 36.332, "latency": 8003.993, "stderr": 2.636, "cost_per_test": 42.619593, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 36.091, "latency": 5104.778, "stderr": 2.429, "cost_per_test": 6.893353, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 36.067, "latency": 4387.047, "stderr": 2.468, "cost_per_test": 2.425297, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 35.629, "latency": 14784.867, "stderr": 2.604, "cost_per_test": 30.247395, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 35.613, "latency": 9422.925, "stderr": 3.058, "cost_per_test": 2.364198, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 35.171, "latency": 4613.696, "stderr": 2.394, "cost_per_test": 40.880803, "turn_metrics": { "tasks_measured": 120, "avg_turns": 309.2, "avg_query_seconds": 10.3, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 34.018, "latency": 11464.514, "stderr": 2.693, "cost_per_test": 0.811766, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 32.053, "latency": 14905.255, "stderr": 2.637, "cost_per_test": 23.900294, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 31.985, "latency": 9130.856, "stderr": 2.386, "cost_per_test": 4.627483, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "stepfun/step-5-preview": { "accuracy": 31.969, "latency": 14495.57, "stderr": 2.628, "cost_per_test": 9.165067, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 31.236, "latency": 13077.791, "stderr": 2.781, "cost_per_test": 0.601193, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 30.333, "latency": 7067.428, "stderr": 2.311, "cost_per_test": 48.506808, "turn_metrics": { "tasks_measured": 120, "avg_turns": 462.5, "avg_query_seconds": 12.4, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 27.16, "latency": 7597.777, "stderr": 2.143, "cost_per_test": 15.070562, "turn_metrics": { "tasks_measured": 120, "avg_turns": 253.2, "avg_query_seconds": 23.5, "median_query_seconds": 14.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 25.871, "latency": 10040.43, "stderr": 2.456, "cost_per_test": 22.86835, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 25.461, "latency": 1656.978, "stderr": 2.134, "cost_per_test": 4.138752, "turn_metrics": { "tasks_measured": 120, "avg_turns": 76.3, "avg_query_seconds": 13.8, "median_query_seconds": 7.5, "avg_ttft_seconds": 4.8, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 23.788, "latency": 2507.905, "stderr": 2.23, "cost_per_test": 9.677001, "turn_metrics": { "tasks_measured": 120, "avg_turns": 120.5, "avg_query_seconds": 17.9, "median_query_seconds": 9.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 23.543, "latency": 10206.969, "stderr": 2.577, "cost_per_test": 27.492529, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/ember-1": { "accuracy": 23.344, "latency": 18009.163, "stderr": 2.901, "cost_per_test": 65.083706, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 19.332, "latency": 6761.8, "stderr": 2.151, "cost_per_test": 10.771789, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 18.833, "latency": 3837.721, "stderr": 1.569, "cost_per_test": 6.652074, "turn_metrics": { "tasks_measured": 120, "avg_turns": 261.2, "avg_query_seconds": 13.5, "median_query_seconds": 7.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 18.629, "latency": 2739.514, "stderr": 2.08, "cost_per_test": 5.536818, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 17.977, "latency": 6170.995, "stderr": 1.675, "cost_per_test": 6.079502, "turn_metrics": { "tasks_measured": 120, "avg_turns": 262.2, "avg_query_seconds": 21.9, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 17.414, "latency": 17876.945, "stderr": 2.396, "cost_per_test": 27.762018, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 17.175, "latency": 1277.58, "stderr": 1.823, "cost_per_test": 6.345007, "turn_metrics": { "tasks_measured": 120, "avg_turns": 173.6, "avg_query_seconds": 6.4, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 17.081, "latency": 2009.347, "stderr": 1.916, "cost_per_test": 4.252549, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 15.47, "latency": 1746.505, "stderr": 1.689, "cost_per_test": 1.108348, "turn_metrics": { "tasks_measured": 120, "avg_turns": 140.5, "avg_query_seconds": 11.1, "median_query_seconds": 5.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 14.386, "latency": 1925.601, "stderr": 1.7, "cost_per_test": 5.22785, "turn_metrics": { "tasks_measured": 120, "avg_turns": 133.6, "avg_query_seconds": 13.0, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 13.981, "latency": 3349.47, "stderr": 1.688, "cost_per_test": 0.268603, "turn_metrics": { "tasks_measured": 120, "avg_turns": 179.7, "avg_query_seconds": 15.5, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 13.916, "latency": 4757.646, "stderr": 2.078, "cost_per_test": 4.987271, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 13.44, "latency": 6689.822, "stderr": 1.683, "cost_per_test": 5.885386, "turn_metrics": { "tasks_measured": 120, "avg_turns": 231.8, "avg_query_seconds": 24.9, "median_query_seconds": 15.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 11.944, "latency": 16988.844, "stderr": 1.837, "cost_per_test": 11.887826, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 11.934, "latency": 4906.501, "stderr": 1.453, "cost_per_test": 8.42404, "turn_metrics": { "tasks_measured": 120, "avg_turns": 370.7, "avg_query_seconds": 8.8, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 11.859, "latency": 2128.98, "stderr": 1.666, "cost_per_test": 0.410777, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 11.625, "latency": 1613.281, "stderr": 1.536, "cost_per_test": 1.981447, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 10.833, "latency": 735.951, "stderr": 1.468, "cost_per_test": 1.521038, "turn_metrics": { "tasks_measured": 120, "avg_turns": 81.1, "avg_query_seconds": 8.2, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 10.358, "latency": 7586.754, "stderr": 1.74, "cost_per_test": 9.744619, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 10.152, "latency": 1377.278, "stderr": 1.905, "cost_per_test": 0.646598, "turn_metrics": { "tasks_measured": 120, "avg_turns": 148.6, "avg_query_seconds": 6.5, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 10.068, "latency": 579.0, "stderr": 1.408, "cost_per_test": 0.949776, "turn_metrics": { "tasks_measured": 120, "avg_turns": 112.0, "avg_query_seconds": 4.2, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 9.88, "latency": 2643.447, "stderr": 1.46, "cost_per_test": 0.11675, "turn_metrics": { "tasks_measured": 120, "avg_turns": 161.1, "avg_query_seconds": 12.4, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.3": { "accuracy": 9.06, "latency": 172.747, "stderr": 1.593, "cost_per_test": 0.360596, "turn_metrics": { "tasks_measured": 120, "avg_turns": 40.0, "avg_query_seconds": 2.9, "median_query_seconds": 1.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 8.521, "latency": 294.005, "stderr": 1.458, "cost_per_test": 0.353141, "turn_metrics": { "tasks_measured": 120, "avg_turns": 62.2, "avg_query_seconds": 3.9, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 8.2, "latency": 1956.927, "stderr": 1.295, "cost_per_test": 2.31844, "turn_metrics": { "tasks_measured": 120, "avg_turns": 110.7, "avg_query_seconds": 15.8, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 8.027, "latency": 4230.282, "stderr": 1.422, "cost_per_test": 0.948698, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 7.354, "latency": 12628.346, "stderr": 1.355, "cost_per_test": 3.522922, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 7.036, "latency": 1604.54, "stderr": 1.452, "cost_per_test": 2.450065, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-2.5": { "accuracy": 6.023, "latency": 760.46, "stderr": 1.398, "cost_per_test": 0.522043, "turn_metrics": { "tasks_measured": 120, "avg_turns": 129.0, "avg_query_seconds": 2.1, "median_query_seconds": 1.2, "avg_ttft_seconds": 1.6, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 5.912, "latency": 1297.768, "stderr": 1.188, "cost_per_test": 0.574959, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 5.875, "latency": 1185.207, "stderr": 1.25, "cost_per_test": 0.478513, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 5.729, "latency": 87.657, "stderr": 1.363, "cost_per_test": 0.052306, "turn_metrics": { "tasks_measured": 120, "avg_turns": 30.1, "avg_query_seconds": 1.8, "median_query_seconds": 1.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 4.991, "latency": 13430.629, "stderr": 1.118, "cost_per_test": 17.48854, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 4.636, "latency": 6272.162, "stderr": 0.929, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 3.935, "latency": 3051.353, "stderr": 1.268, "cost_per_test": 0.280724, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 3.402, "latency": 1654.93, "stderr": 0.728, "cost_per_test": 0.409803, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 2.209, "latency": 17903.02, "stderr": 0.576, "cost_per_test": 23.527047, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k3": { "accuracy": 1.465, "latency": 18077.226, "stderr": 0.524, "cost_per_test": 14.391633, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 1.063, "latency": 1876.841, "stderr": 0.583, "cost_per_test": 0.08168, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.418, "latency": 404.445, "stderr": 0.146, "cost_per_test": 0.264092, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 17910.443, "stderr": 0.0, "cost_per_test": 0.051099, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null } }, "cobol": { "grok/grok-4.5": { "accuracy": 70.0, "latency": 664.305, "stderr": 15.275, "cost_per_test": 1.062967, "turn_metrics": { "tasks_measured": 10, "avg_turns": 39.5, "avg_query_seconds": 12.7, "median_query_seconds": 6.4, "avg_ttft_seconds": 7.3, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 70.0, "latency": 897.888, "stderr": 15.275, "cost_per_test": 0.255782, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 70.0, "latency": 1100.355, "stderr": 15.275, "cost_per_test": 4.606544, "turn_metrics": { "tasks_measured": 10, "avg_turns": 34.3, "avg_query_seconds": 15.3, "median_query_seconds": 8.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 70.0, "latency": 1233.694, "stderr": 15.275, "cost_per_test": 1.550605, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 70.0, "latency": 1274.912, "stderr": 15.275, "cost_per_test": 2.227453, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 70.0, "latency": 2008.956, "stderr": 15.275, "cost_per_test": 1.178122, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 70.0, "latency": 2098.457, "stderr": 15.275, "cost_per_test": 6.754881, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 70.0, "latency": 2115.658, "stderr": 15.275, "cost_per_test": 5.951258, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 70.0, "latency": 2340.697, "stderr": 15.275, "cost_per_test": 3.151116, "turn_metrics": { "tasks_measured": 10, "avg_turns": 48.2, "avg_query_seconds": 17.0, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 70.0, "latency": 2563.514, "stderr": 15.275, "cost_per_test": 10.578392, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 70.0, "latency": 3105.252, "stderr": 15.275, "cost_per_test": 0.339506, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 70.0, "latency": 3456.844, "stderr": 15.275, "cost_per_test": 0.328209, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 70.0, "latency": 3466.953, "stderr": 15.275, "cost_per_test": 22.908316, "turn_metrics": { "tasks_measured": 10, "avg_turns": 182.1, "avg_query_seconds": 14.9, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 70.0, "latency": 3955.804, "stderr": 15.275, "cost_per_test": 1.015163, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse-wonder-full": { "accuracy": 70.0, "latency": 4255.027, "stderr": 15.275, "cost_per_test": 6.193699, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 70.0, "latency": 4570.89, "stderr": 15.275, "cost_per_test": 4.247581, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 70.0, "latency": 4632.139, "stderr": 15.275, "cost_per_test": 5.331586, "turn_metrics": { "tasks_measured": 10, "avg_turns": 104.6, "avg_query_seconds": 35.0, "median_query_seconds": 16.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 70.0, "latency": 5017.804, "stderr": 15.275, "cost_per_test": 17.499254, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 70.0, "latency": 5072.704, "stderr": 15.275, "cost_per_test": 22.947267, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 70.0, "latency": 6152.06, "stderr": 15.275, "cost_per_test": 0.331265, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 70.0, "latency": 6191.772, "stderr": 15.275, "cost_per_test": 2.624357, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 70.0, "latency": 6921.616, "stderr": 15.275, "cost_per_test": 0.170673, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 70.0, "latency": 7101.336, "stderr": 15.275, "cost_per_test": 19.396217, "turn_metrics": { "tasks_measured": 10, "avg_turns": 192.7, "avg_query_seconds": 15.8, "median_query_seconds": 9.5, "avg_ttft_seconds": 15.5, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 70.0, "latency": 10179.948, "stderr": 15.275, "cost_per_test": 8.896834, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 70.0, "latency": 16134.188, "stderr": 15.275, "cost_per_test": 14.803444, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 68.571, "latency": 1061.3, "stderr": 15.028, "cost_per_test": 1.99524, "turn_metrics": { "tasks_measured": 10, "avg_turns": 47.7, "avg_query_seconds": 17.3, "median_query_seconds": 9.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 68.571, "latency": 1160.346, "stderr": 15.028, "cost_per_test": 5.523588, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 68.571, "latency": 1186.716, "stderr": 15.028, "cost_per_test": 1.630862, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 68.571, "latency": 1439.67, "stderr": 15.028, "cost_per_test": 2.380724, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 68.571, "latency": 1971.492, "stderr": 15.028, "cost_per_test": 3.505601, "turn_metrics": { "tasks_measured": 10, "avg_turns": 120.4, "avg_query_seconds": 10.2, "median_query_seconds": 5.1, "avg_ttft_seconds": 5.3, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 68.571, "latency": 2118.518, "stderr": 15.028, "cost_per_test": 2.848177, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 68.571, "latency": 2713.85, "stderr": 15.028, "cost_per_test": 8.510144, "turn_metrics": { "tasks_measured": 10, "avg_turns": 134.6, "avg_query_seconds": 17.6, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 68.571, "latency": 3536.152, "stderr": 15.028, "cost_per_test": 15.800701, "turn_metrics": { "tasks_measured": 10, "avg_turns": 100.2, "avg_query_seconds": 29.5, "median_query_seconds": 14.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 68.571, "latency": 4094.2, "stderr": 15.028, "cost_per_test": 13.382972, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 68.571, "latency": 7459.618, "stderr": 15.028, "cost_per_test": 3.378437, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 68.571, "latency": 7651.975, "stderr": 15.028, "cost_per_test": 5.361607, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 65.714, "latency": 1333.918, "stderr": 14.938, "cost_per_test": 3.406297, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 61.25, "latency": 3713.208, "stderr": 15.861, "cost_per_test": 2.507493, "turn_metrics": { "tasks_measured": 10, "avg_turns": 128.7, "avg_query_seconds": 21.4, "median_query_seconds": 9.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.6": { "accuracy": 60.0, "latency": 3597.623, "stderr": 16.33, "cost_per_test": 3.517206, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/ember-1": { "accuracy": 60.0, "latency": 5638.286, "stderr": 16.33, "cost_per_test": 9.192215, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 60.0, "latency": 7571.461, "stderr": 16.33, "cost_per_test": 12.309516, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 60.0, "latency": 7940.872, "stderr": 16.33, "cost_per_test": 2.932338, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 60.0, "latency": 11442.446, "stderr": 16.33, "cost_per_test": 1.94652, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 60.0, "latency": 12803.716, "stderr": 16.33, "cost_per_test": 22.611601, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 60.0, "latency": 13296.382, "stderr": 16.33, "cost_per_test": 6.489161, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-luna": { "accuracy": 58.571, "latency": 1061.818, "stderr": 16.0, "cost_per_test": 0.082651, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 58.571, "latency": 3409.251, "stderr": 16.0, "cost_per_test": 0.541149, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 58.393, "latency": 1712.395, "stderr": 15.334, "cost_per_test": 0.959022, "turn_metrics": { "tasks_measured": 10, "avg_turns": 112.6, "avg_query_seconds": 13.3, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 57.143, "latency": 3081.279, "stderr": 15.793, "cost_per_test": 2.209705, "turn_metrics": { "tasks_measured": 10, "avg_turns": 125.7, "avg_query_seconds": 19.9, "median_query_seconds": 8.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 55.455, "latency": 841.077, "stderr": 15.478, "cost_per_test": 3.865467, "turn_metrics": { "tasks_measured": 10, "avg_turns": 135.8, "avg_query_seconds": 4.6, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 53.611, "latency": 4402.526, "stderr": 12.87, "cost_per_test": 6.935688, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 50.0, "latency": 14891.833, "stderr": 16.667, "cost_per_test": 16.130675, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 47.662, "latency": 1411.857, "stderr": 15.949, "cost_per_test": 0.435492, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 46.571, "latency": 2496.289, "stderr": 15.655, "cost_per_test": 2.84903, "turn_metrics": { "tasks_measured": 10, "avg_turns": 147.0, "avg_query_seconds": 12.0, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 44.286, "latency": 1683.159, "stderr": 15.714, "cost_per_test": 0.147392, "turn_metrics": { "tasks_measured": 10, "avg_turns": 75.2, "avg_query_seconds": 16.9, "median_query_seconds": 6.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 43.896, "latency": 2993.546, "stderr": 15.162, "cost_per_test": 3.018516, "turn_metrics": { "tasks_measured": 10, "avg_turns": 197.7, "avg_query_seconds": 8.9, "median_query_seconds": 5.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 40.0, "latency": 4411.994, "stderr": 16.33, "cost_per_test": 31.08541, "turn_metrics": { "tasks_measured": 7, "avg_turns": 125.6, "avg_query_seconds": 28.6, "median_query_seconds": 16.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 36.964, "latency": 1037.503, "stderr": 14.817, "cost_per_test": 0.663085, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 36.753, "latency": 674.444, "stderr": 15.105, "cost_per_test": 1.895374, "turn_metrics": { "tasks_measured": 10, "avg_turns": 83.4, "avg_query_seconds": 6.5, "median_query_seconds": 4.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 27.273, "latency": 1168.186, "stderr": 14.084, "cost_per_test": 0.060959, "turn_metrics": { "tasks_measured": 10, "avg_turns": 82.1, "avg_query_seconds": 10.3, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 27.143, "latency": 1063.103, "stderr": 14.038, "cost_per_test": 0.857472, "turn_metrics": { "tasks_measured": 10, "avg_turns": 46.2, "avg_query_seconds": 20.6, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 26.055, "latency": 2280.389, "stderr": 12.866, "cost_per_test": 4.677675, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 17.403, "latency": 1287.566, "stderr": 10.464, "cost_per_test": 1.787255, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 15.844, "latency": 2201.033, "stderr": 10.607, "cost_per_test": 0.444589, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 6.964, "latency": 1733.588, "stderr": 5.712, "cost_per_test": 0.665203, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 5.714, "latency": 3065.567, "stderr": 5.714, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 5.536, "latency": 11929.423, "stderr": 4.329, "cost_per_test": 22.029403, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 4.286, "latency": 905.439, "stderr": 4.286, "cost_per_test": 0.410702, "turn_metrics": { "tasks_measured": 10, "avg_turns": 129.2, "avg_query_seconds": 6.2, "median_query_seconds": 3.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 4.107, "latency": 1449.663, "stderr": 2.989, "cost_per_test": 0.168875, "turn_metrics": { "tasks_measured": 10, "avg_turns": 79.9, "avg_query_seconds": 10.3, "median_query_seconds": 5.5, "avg_ttft_seconds": 2.4, "output_time_split": { "reasoning_pct": 42.5, "content_pct": 3.2, "tool_call_pct": 54.3 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 3.766, "latency": 2591.998, "stderr": 2.9, "cost_per_test": 0.861542, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 1.25, "latency": 107.549, "stderr": 1.25, "cost_per_test": 0.055194, "turn_metrics": { "tasks_measured": 10, "avg_turns": 28.5, "avg_query_seconds": 2.1, "median_query_seconds": 1.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 1.25, "latency": 1383.042, "stderr": 1.25, "cost_per_test": 3.443127, "turn_metrics": { "tasks_measured": 10, "avg_turns": 103.5, "avg_query_seconds": 12.1, "median_query_seconds": 4.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 203.671, "stderr": 0.0, "cost_per_test": 0.585897, "turn_metrics": { "tasks_measured": 10, "avg_turns": 63.5, "avg_query_seconds": 2.6, "median_query_seconds": 1.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 254.923, "stderr": 0.0, "cost_per_test": 0.300925, "turn_metrics": { "tasks_measured": 10, "avg_turns": 41.1, "avg_query_seconds": 6.0, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 281.358, "stderr": 0.0, "cost_per_test": 0.365668, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 637.86, "stderr": 0.0, "cost_per_test": 1.076978, "turn_metrics": { "tasks_measured": 10, "avg_turns": 126.4, "avg_query_seconds": 3.9, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 1264.931, "stderr": 0.0, "cost_per_test": 0.225898, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 1614.952, "stderr": 0.0, "cost_per_test": 0.94276, "turn_metrics": { "tasks_measured": 10, "avg_turns": 146.6, "avg_query_seconds": 2.8, "median_query_seconds": 1.9, "avg_ttft_seconds": 2.5, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 18007.728, "stderr": 0.0, "cost_per_test": 0.087152, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null } }, "code_quality": { "anthropic/claude-sonnet-5": { "accuracy": 4.898, "latency": 7026.544, "stderr": 0.015, "cost_per_test": 35.310438, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 4.825, "latency": 4971.306, "stderr": 0.022, "cost_per_test": 6.23191, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 4.75, "latency": 10268.729, "stderr": 0.021, "cost_per_test": 60.509587, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 4.7, "latency": 4498.693, "stderr": 0.025, "cost_per_test": 30.514184, "turn_metrics": { "tasks_measured": 120, "avg_turns": 193.2, "avg_query_seconds": 21.5, "median_query_seconds": 13.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 4.601, "latency": 12004.483, "stderr": 0.034, "cost_per_test": 75.826721, "turn_metrics": { "tasks_measured": 120, "avg_turns": 558.0, "avg_query_seconds": 12.5, "median_query_seconds": 7.3, "avg_ttft_seconds": 12.1, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 4.588, "latency": 4327.01, "stderr": 0.034, "cost_per_test": 36.387681, "turn_metrics": { "tasks_measured": 120, "avg_turns": 309.2, "avg_query_seconds": 10.3, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 4.541, "latency": 11925.532, "stderr": 0.032, "cost_per_test": 112.970791, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 4.513, "latency": 15610.474, "stderr": 0.036, "cost_per_test": 70.973455, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 4.502, "latency": 13633.637, "stderr": 0.034, "cost_per_test": 24.909755, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 4.496, "latency": 5781.916, "stderr": 0.035, "cost_per_test": 6.505708, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 4.469, "latency": 3510.003, "stderr": 0.033, "cost_per_test": 44.364548, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 4.442, "latency": 6559.388, "stderr": 0.029, "cost_per_test": 112.098109, "turn_metrics": { "tasks_measured": 120, "avg_turns": 267.4, "avg_query_seconds": 21.7, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 4.423, "latency": 7994.848, "stderr": 0.036, "cost_per_test": 3.405133, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 4.411, "latency": 6325.865, "stderr": 0.035, "cost_per_test": 0.944694, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "stepfun/step-5-preview": { "accuracy": 4.379, "latency": 12856.895, "stderr": 0.036, "cost_per_test": 7.606885, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-sol": { "accuracy": 4.375, "latency": 5136.155, "stderr": 0.046, "cost_per_test": 15.676945, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 4.355, "latency": 12331.871, "stderr": 0.047, "cost_per_test": 3.128822, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 4.267, "latency": 5054.737, "stderr": 0.049, "cost_per_test": 14.210071, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 4.255, "latency": 16065.728, "stderr": 0.054, "cost_per_test": 10.53816, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse-wonder-full": { "accuracy": 4.244, "latency": 13956.531, "stderr": 0.045, "cost_per_test": 38.410556, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 4.243, "latency": 10136.401, "stderr": 0.038, "cost_per_test": 0.691641, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 4.197, "latency": 17150.223, "stderr": 0.078, "cost_per_test": 21.677954, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 4.133, "latency": 17441.256, "stderr": 0.055, "cost_per_test": 24.522375, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.7": { "accuracy": 4.105, "latency": 4141.411, "stderr": 0.046, "cost_per_test": 36.55237, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 4.094, "latency": 11538.747, "stderr": 0.047, "cost_per_test": 0.493563, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.5": { "accuracy": 4.078, "latency": 1408.81, "stderr": 0.042, "cost_per_test": 3.369806, "turn_metrics": { "tasks_measured": 120, "avg_turns": 76.3, "avg_query_seconds": 13.8, "median_query_seconds": 7.5, "avg_ttft_seconds": 4.8, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.6": { "accuracy": 4.077, "latency": 5429.239, "stderr": 0.037, "cost_per_test": 14.575491, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-wonder": { "accuracy": 4.061, "latency": 10267.754, "stderr": 0.054, "cost_per_test": 69.535349, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 4.032, "latency": 12726.884, "stderr": 0.042, "cost_per_test": 18.581309, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 4.02, "latency": 4061.108, "stderr": 0.068, "cost_per_test": 4.616854, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 3.997, "latency": 5979.033, "stderr": 0.032, "cost_per_test": 38.507642, "turn_metrics": { "tasks_measured": 120, "avg_turns": 462.5, "avg_query_seconds": 12.4, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 3.992, "latency": 7843.507, "stderr": 0.055, "cost_per_test": 1.858025, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 3.981, "latency": 6856.367, "stderr": 0.052, "cost_per_test": 12.635818, "turn_metrics": { "tasks_measured": 120, "avg_turns": 253.2, "avg_query_seconds": 23.5, "median_query_seconds": 14.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/ember-1": { "accuracy": 3.959, "latency": 14916.444, "stderr": 0.054, "cost_per_test": 51.110833, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 3.948, "latency": 2914.132, "stderr": 0.066, "cost_per_test": 8.127254, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 3.948, "latency": 7700.455, "stderr": 0.043, "cost_per_test": 3.605899, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 3.899, "latency": 15450.785, "stderr": 0.118, "cost_per_test": 13.871104, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 3.847, "latency": 3551.876, "stderr": 0.06, "cost_per_test": 24.541448, "turn_metrics": { "tasks_measured": 120, "avg_turns": 143.1, "avg_query_seconds": 12.2, "median_query_seconds": 8.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 3.752, "latency": 3023.191, "stderr": 0.06, "cost_per_test": 0.600876, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 3.742, "latency": 9443.316, "stderr": 0.053, "cost_per_test": 18.491664, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 3.718, "latency": 4428.262, "stderr": 0.083, "cost_per_test": 7.072659, "turn_metrics": { "tasks_measured": 120, "avg_turns": 370.7, "avg_query_seconds": 8.8, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 3.664, "latency": 9520.131, "stderr": 0.055, "cost_per_test": 21.464006, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 3.654, "latency": 1594.162, "stderr": 0.067, "cost_per_test": 0.416225, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 3.632, "latency": 2146.253, "stderr": 0.085, "cost_per_test": 7.756561, "turn_metrics": { "tasks_measured": 120, "avg_turns": 120.5, "avg_query_seconds": 17.9, "median_query_seconds": 9.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 3.574, "latency": 7891.461, "stderr": 0.048, "cost_per_test": 57.822126, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 3.547, "latency": 5945.669, "stderr": 0.065, "cost_per_test": 5.040913, "turn_metrics": { "tasks_measured": 120, "avg_turns": 231.8, "avg_query_seconds": 24.9, "median_query_seconds": 15.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.1": { "accuracy": 3.544, "latency": 3502.363, "stderr": 0.061, "cost_per_test": 5.701313, "turn_metrics": { "tasks_measured": 120, "avg_turns": 261.2, "avg_query_seconds": 13.5, "median_query_seconds": 7.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 3.518, "latency": 1789.961, "stderr": 0.075, "cost_per_test": 4.781669, "turn_metrics": { "tasks_measured": 120, "avg_turns": 133.6, "avg_query_seconds": 13.0, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.5": { "accuracy": 3.498, "latency": 1894.953, "stderr": 0.048, "cost_per_test": 6.442253, "turn_metrics": { "tasks_measured": 120, "avg_turns": 87.4, "avg_query_seconds": 16.7, "median_query_seconds": 8.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 3.437, "latency": 1531.852, "stderr": 0.045, "cost_per_test": 1.932899, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 3.401, "latency": 1737.978, "stderr": 0.036, "cost_per_test": 1.071016, "turn_metrics": { "tasks_measured": 120, "avg_turns": 140.5, "avg_query_seconds": 11.1, "median_query_seconds": 5.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 3.382, "latency": 2932.892, "stderr": 0.061, "cost_per_test": 0.2383, "turn_metrics": { "tasks_measured": 120, "avg_turns": 179.7, "avg_query_seconds": 15.5, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 3.363, "latency": 2146.993, "stderr": 0.063, "cost_per_test": 0.41923, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 3.348, "latency": 593.715, "stderr": 0.036, "cost_per_test": 0.981576, "turn_metrics": { "tasks_measured": 120, "avg_turns": 112.0, "avg_query_seconds": 4.2, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 3.299, "latency": 5404.829, "stderr": 0.073, "cost_per_test": 8.930416, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 3.274, "latency": 3820.711, "stderr": 0.058, "cost_per_test": 0.926909, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 3.245, "latency": 5470.513, "stderr": 0.045, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 3.195, "latency": 3514.758, "stderr": 0.065, "cost_per_test": 1.882918, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 3.149, "latency": 5398.566, "stderr": 0.071, "cost_per_test": 5.112053, "turn_metrics": { "tasks_measured": 120, "avg_turns": 262.2, "avg_query_seconds": 21.9, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.3": { "accuracy": 3.098, "latency": 180.478, "stderr": 0.131, "cost_per_test": 0.416921, "turn_metrics": { "tasks_measured": 120, "avg_turns": 40.0, "avg_query_seconds": 2.9, "median_query_seconds": 1.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 3.053, "latency": 1733.471, "stderr": 0.076, "cost_per_test": 1.953198, "turn_metrics": { "tasks_measured": 120, "avg_turns": 110.7, "avg_query_seconds": 15.8, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 3.037, "latency": 1866.928, "stderr": 0.093, "cost_per_test": 3.784593, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 2.987, "latency": 13055.328, "stderr": 0.062, "cost_per_test": 18.623756, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 2.966, "latency": 1259.318, "stderr": 0.037, "cost_per_test": 0.587624, "turn_metrics": { "tasks_measured": 120, "avg_turns": 148.6, "avg_query_seconds": 6.5, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 2.805, "latency": 2351.314, "stderr": 0.085, "cost_per_test": 4.560329, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 2.789, "latency": 1168.454, "stderr": 0.073, "cost_per_test": 5.725122, "turn_metrics": { "tasks_measured": 120, "avg_turns": 173.6, "avg_query_seconds": 6.4, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 2.788, "latency": 2274.632, "stderr": 0.077, "cost_per_test": 0.102802, "turn_metrics": { "tasks_measured": 120, "avg_turns": 161.1, "avg_query_seconds": 12.4, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "inception/mercury-2.5": { "accuracy": 2.737, "latency": 974.083, "stderr": 0.067, "cost_per_test": 0.627223, "turn_metrics": { "tasks_measured": 120, "avg_turns": 129.0, "avg_query_seconds": 2.1, "median_query_seconds": 1.2, "avg_ttft_seconds": 1.6, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 2.664, "latency": 92.63, "stderr": 0.038, "cost_per_test": 0.053028, "turn_metrics": { "tasks_measured": 120, "avg_turns": 30.1, "avg_query_seconds": 1.8, "median_query_seconds": 1.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 2.662, "latency": 284.234, "stderr": 0.047, "cost_per_test": 0.340087, "turn_metrics": { "tasks_measured": 120, "avg_turns": 62.2, "avg_query_seconds": 3.9, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 2.652, "latency": 1322.302, "stderr": 0.05, "cost_per_test": 0.525186, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 2.608, "latency": 1770.047, "stderr": 0.103, "cost_per_test": 0.103478, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 2.562, "latency": 2604.747, "stderr": 0.071, "cost_per_test": 0.267017, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 2.538, "latency": 720.574, "stderr": 0.039, "cost_per_test": 1.614622, "turn_metrics": { "tasks_measured": 120, "avg_turns": 81.1, "avg_query_seconds": 8.2, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 2.463, "latency": 6790.697, "stderr": 0.047, "cost_per_test": 9.042387, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 2.292, "latency": 1232.702, "stderr": 0.058, "cost_per_test": 0.596991, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 2.247, "latency": 1773.502, "stderr": 0.059, "cost_per_test": 3.006967, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 2.167, "latency": 373.673, "stderr": 0.075, "cost_per_test": 0.289486, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 17934.764, "stderr": 0.0, "cost_per_test": 0.060112, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null } } } }