{ "metadata": { "benchmark": "Terminal-Bench 2.1", "slug": "terminal-bench-2-1", "description": "State-of-the-art set of difficult terminal-based tasks", "benchmark_id": "terminal_bench_2_1", "family": "terminal_bench", "version": "2.1", "updated": "2026-10-01", "dataset_type": "public", "industry": "coding", "tasks": { "overall": "Overall", "easy": "Easy", "medium": "Medium", "hard": "Hard" }, "models": [ "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.7-plus", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "ant/ling-3.0-flash-2607", "ant/ling-3.0-flash-af-rc3", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-4-8-claude-code", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "cohere/command-a-plus-05-2026", "cursor/composer-2.5", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "fireworks/nemotron-lightning-3p5-30b-a3b", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-4.7", "inception/mercury-2.5", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k2.7-code", "kimi/kimi-k3", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/mistral-medium-3.5", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.5-codex", "openai/gpt-5.5-factory", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "poolside/laguna-m.1", "poolside/laguna-xs.2", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.1", "zai/glm-5.2", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": true, "visible": true, "use_cost_per_test": true, "token_tasks_measured": 89, "harness": "Terminus 2", "runner": "external", "mode": "agentic", "archived": true, "partner": false, "total_models": 76 }, "tasks": { "overall": { "anthropic/claude-opus-5-5": { "accuracy": 87.64, "latency": 383.079, "stderr": 1.716, "cost_per_test": 0.496033, "token_totals": { "input_tokens": 22792900, "output_tokens": 1198947, "reasoning_tokens": 351772, "cache_read_tokens": 20604830, "cache_write_tokens": 2181727 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 87, "avg_turns": 12.8, "avg_query_seconds": 10.2, "median_query_seconds": 4.1, "avg_ttft_seconds": 4.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 87.266, "latency": 274.44, "stderr": 0.375, "cost_per_test": 1.339493, "token_totals": { "input_tokens": 24715799, "output_tokens": 1309769, "reasoning_tokens": 764679, "cache_read_tokens": 21492398, "cache_write_tokens": null }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 12.6, "avg_query_seconds": 17.4, "median_query_seconds": 8.4, "avg_ttft_seconds": 9.4, "output_time_split": { "reasoning_pct": 18.9, "content_pct": 81.1, "tool_call_pct": 0.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 85.768, "latency": 372.819, "stderr": 1.35, "cost_per_test": 1.023319, "token_totals": { "input_tokens": 29114624, "output_tokens": 1324119, "reasoning_tokens": 721900, "cache_read_tokens": 20938069, "cache_write_tokens": null }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 13.8, "avg_query_seconds": 15.9, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 85.019, "latency": 500.355, "stderr": 0.991, "cost_per_test": 3.074796, "token_totals": { "input_tokens": 21892452, "output_tokens": 965107, "reasoning_tokens": 472161, "cache_read_tokens": 19902668, "cache_write_tokens": 1985257 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 88, "avg_turns": 13.2, "avg_query_seconds": 14.5, "median_query_seconds": 7.5, "avg_ttft_seconds": 10.0, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 84.644, "latency": 752.932, "stderr": 0.991, "cost_per_test": 0.886936, "token_totals": { "input_tokens": 35206371, "output_tokens": 1733729, "reasoning_tokens": 1113936, "cache_read_tokens": 32076924, "cache_write_tokens": 3126721 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 15.1, "avg_query_seconds": 18.0, "median_query_seconds": 7.8, "avg_ttft_seconds": 13.7, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 83.146, "latency": 298.876, "stderr": 1.297, "cost_per_test": 0.373024, "token_totals": { "input_tokens": 53182967, "output_tokens": 1568343, "reasoning_tokens": 998593, "cache_read_tokens": 50178777, "cache_write_tokens": 2943219 }, "token_tasks_measured": 89, "turn_metrics": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 83.146, "latency": 465.828, "stderr": 1.716, "cost_per_test": 0.625494, "token_totals": { "input_tokens": 47223436, "output_tokens": 3585507, "reasoning_tokens": 379696, "cache_read_tokens": 43257785, "cache_write_tokens": 3960025 }, "token_tasks_measured": 89, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 81.273, "latency": 554.797, "stderr": 0.375, "cost_per_test": 1.544972, "token_totals": { "input_tokens": 325795812, "output_tokens": 4741834, "reasoning_tokens": 2933371, "cache_read_tokens": 286485175, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 62.6, "avg_query_seconds": 4.5, "median_query_seconds": 3.0, "avg_ttft_seconds": 3.0, "output_time_split": { "reasoning_pct": 43.3, "content_pct": 56.7, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 80.899, "latency": 608.575, "stderr": 0.649, "cost_per_test": 0.341034, "token_totals": { "input_tokens": 20494904, "output_tokens": 1244660, "reasoning_tokens": 843947, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 11.9, "avg_query_seconds": 25.3, "median_query_seconds": 8.6, "avg_ttft_seconds": 1.7, "output_time_split": { "reasoning_pct": 69.9, "content_pct": 30.1, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 80.524, "latency": 504.537, "stderr": 1.35, "cost_per_test": 1.429025, "token_totals": { "input_tokens": 21092499, "output_tokens": 1498447, "reasoning_tokens": null, "cache_read_tokens": 18355304, "cache_write_tokens": 2613456 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 11.3, "avg_query_seconds": 21.2, "median_query_seconds": 7.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 79.026, "latency": 355.751, "stderr": 0.991, "cost_per_test": 0.053767, "token_totals": { "input_tokens": 67560156, "output_tokens": 2052883, "reasoning_tokens": 1203057, "cache_read_tokens": 62168050, "cache_write_tokens": null }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 19.6, "avg_query_seconds": 8.2, "median_query_seconds": 5.0, "avg_ttft_seconds": 3.0, "output_time_split": { "reasoning_pct": 52.0, "content_pct": 48.0, "tool_call_pct": 0.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 79.026, "latency": 731.78, "stderr": 0.991, "cost_per_test": 0.59671, "token_totals": { "input_tokens": 198539978, "output_tokens": 3621191, "reasoning_tokens": 2589082, "cache_read_tokens": 191325278, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 88, "avg_turns": 41.3, "avg_query_seconds": 8.6, "median_query_seconds": 3.6, "avg_ttft_seconds": 7.6, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 78.277, "latency": 642.908, "stderr": 2.085, "cost_per_test": 0.453642, "token_totals": { "input_tokens": 37989503, "output_tokens": 2301698, "reasoning_tokens": 1606577, "cache_read_tokens": 32943360, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 20.1, "avg_query_seconds": 20.6, "median_query_seconds": 5.1, "avg_ttft_seconds": 14.6, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 77.528, "latency": 492.202, "stderr": 0.649, "cost_per_test": 1.022219, "token_totals": { "input_tokens": 195502339, "output_tokens": 3101625, "reasoning_tokens": 1363982, "cache_read_tokens": 167065334, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 44.5, "avg_query_seconds": 5.9, "median_query_seconds": 3.6, "avg_ttft_seconds": 4.5, "output_time_split": { "reasoning_pct": 28.4, "content_pct": 71.6, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 77.528, "latency": 519.739, "stderr": 2.247, "cost_per_test": 0.472552, "token_totals": { "input_tokens": 42753712, "output_tokens": 2043151, "reasoning_tokens": 1345891, "cache_read_tokens": 37760047, "cache_write_tokens": null }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 15.5, "avg_query_seconds": 17.5, "median_query_seconds": 9.0, "avg_ttft_seconds": 5.3, "output_time_split": { "reasoning_pct": 48.1, "content_pct": 51.9, "tool_call_pct": 0.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 76.404, "latency": 427.412, "stderr": 0.649, "cost_per_test": 0.741883, "token_totals": { "input_tokens": 16386293, "output_tokens": 1376238, "reasoning_tokens": 851034, "cache_read_tokens": 12709120, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 11.7, "avg_query_seconds": 25.8, "median_query_seconds": 13.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 76.404, "latency": 809.733, "stderr": 1.716, "cost_per_test": 0.02332, "token_totals": { "input_tokens": 77103048, "output_tokens": 3904513, "reasoning_tokens": 3173128, "cache_read_tokens": 71517419, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 87, "avg_turns": 19.8, "avg_query_seconds": 18.6, "median_query_seconds": 6.1, "avg_ttft_seconds": 2.4, "output_time_split": { "reasoning_pct": 88.4, "content_pct": 11.6, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 74.532, "latency": 604.506, "stderr": 1.633, "cost_per_test": 0.098877, "token_totals": { "input_tokens": 56873906, "output_tokens": 6392867, "reasoning_tokens": 5656496, "cache_read_tokens": 54195840, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 87, "avg_turns": 23.9, "avg_query_seconds": 16.7, "median_query_seconds": 8.2, "avg_ttft_seconds": 1.6, "output_time_split": { "reasoning_pct": 92.3, "content_pct": 7.7, "tool_call_pct": 0.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 74.532, "latency": 635.043, "stderr": 2.085, "cost_per_test": 0.534595, "token_totals": { "input_tokens": 74219799, "output_tokens": 2321306, "reasoning_tokens": null, "cache_read_tokens": 70230688, "cache_write_tokens": 3987009 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 88, "avg_turns": 22.3, "avg_query_seconds": 13.4, "median_query_seconds": 5.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 74.157, "latency": 373.223, "stderr": 1.124, "cost_per_test": 0.868179, "token_totals": { "input_tokens": 135043681, "output_tokens": 2323941, "reasoning_tokens": 196148, "cache_read_tokens": 108305986, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 38.2, "avg_query_seconds": 4.7, "median_query_seconds": 2.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 73.783, "latency": 552.153, "stderr": 1.633, "cost_per_test": 1.174404, "token_totals": { "input_tokens": 224181261, "output_tokens": 3381533, "reasoning_tokens": 1807265, "cache_read_tokens": 190452944, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 43.2, "avg_query_seconds": 6.1, "median_query_seconds": 4.5, "avg_ttft_seconds": 3.7, "output_time_split": { "reasoning_pct": 45.9, "content_pct": 54.1, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 73.408, "latency": 797.952, "stderr": 1.498, "cost_per_test": 1.076601, "token_totals": { "input_tokens": 98794136, "output_tokens": 2621813, "reasoning_tokens": 1801692, "cache_read_tokens": 78833280, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 87, "avg_turns": 27.0, "avg_query_seconds": 18.2, "median_query_seconds": 8.3, "avg_ttft_seconds": 2.7, "output_time_split": { "reasoning_pct": 70.9, "content_pct": 29.1, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 73.034, "latency": 425.094, "stderr": 1.716, "cost_per_test": 0.029222, "token_totals": { "input_tokens": 68771981, "output_tokens": 2862029, "reasoning_tokens": 2313628, "cache_read_tokens": 64567254, "cache_write_tokens": 4144652 }, "token_tasks_measured": 89, "turn_metrics": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 72.285, "latency": 857.636, "stderr": 0.375, "cost_per_test": 0.663249, "token_totals": { "input_tokens": 153551412, "output_tokens": 3070064, "reasoning_tokens": 2095052, "cache_read_tokens": 132688964, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 35.3, "avg_query_seconds": 15.2, "median_query_seconds": 6.9, "avg_ttft_seconds": 4.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 71.91, "latency": 929.902, "stderr": 0.649, "cost_per_test": 2.409772, "token_totals": { "input_tokens": 84371822, "output_tokens": 5125339, "reasoning_tokens": null, "cache_read_tokens": 76658876, "cache_write_tokens": 7553615 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 20.8, "avg_query_seconds": 34.0, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 71.536, "latency": 786.604, "stderr": 1.982, "cost_per_test": 0.30672, "token_totals": { "input_tokens": 46365317, "output_tokens": 2447978, "reasoning_tokens": 1787588, "cache_read_tokens": 42442496, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 15.6, "avg_query_seconds": 33.7, "median_query_seconds": 16.1, "avg_ttft_seconds": 5.2, "output_time_split": { "reasoning_pct": 84.8, "content_pct": 15.2, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 70.787, "latency": 410.249, "stderr": 1.124, "cost_per_test": 0.582677, "token_totals": { "input_tokens": 33939400, "output_tokens": 2667201, "reasoning_tokens": 2048217, "cache_read_tokens": 26933702, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 14.6, "avg_query_seconds": 17.8, "median_query_seconds": 9.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8-claude-code": { "accuracy": 69.663, "latency": 359.967, "stderr": 4.901, "cost_per_test": 0.968098, "token_totals": { "input_tokens": 62415811, "output_tokens": 1520645, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "Claude Code" }, "meta/muse_spark_1_2": { "accuracy": 69.663, "latency": 879.273, "stderr": 0.649, "cost_per_test": 0.499472, "token_totals": { "input_tokens": 131957427, "output_tokens": 3332616, "reasoning_tokens": 2186108, "cache_read_tokens": 122415847, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 31.4, "avg_query_seconds": 16.0, "median_query_seconds": 6.7, "avg_ttft_seconds": 14.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 69.288, "latency": 624.772, "stderr": 0.991, "cost_per_test": 0.417396, "token_totals": { "input_tokens": 138955979, "output_tokens": 3124998, "reasoning_tokens": 1933296, "cache_read_tokens": 136207264, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 30.7, "avg_query_seconds": 10.8, "median_query_seconds": 5.5, "avg_ttft_seconds": 7.8, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 68.539, "latency": 816.587, "stderr": 1.297, "cost_per_test": 1.977087, "token_totals": { "input_tokens": 109753697, "output_tokens": 3484336, "reasoning_tokens": null, "cache_read_tokens": 103807239, "cache_write_tokens": 5773113 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 87, "avg_turns": 26.4, "avg_query_seconds": 19.6, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 67.79, "latency": 624.415, "stderr": 4.416, "cost_per_test": 0.283943, "token_totals": { "input_tokens": 26039357, "output_tokens": 1388617, "reasoning_tokens": 651797, "cache_read_tokens": 23426304, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 86, "avg_turns": 15.0, "avg_query_seconds": 25.1, "median_query_seconds": 9.2, "avg_ttft_seconds": 12.1, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.2": { "accuracy": 67.79, "latency": 820.506, "stderr": 0.991, "cost_per_test": 0.425184, "token_totals": { "input_tokens": 58211167, "output_tokens": 3186166, "reasoning_tokens": 2550689, "cache_read_tokens": 50590720, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 16.7, "avg_query_seconds": 41.7, "median_query_seconds": 20.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 48000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 67.79, "latency": 939.123, "stderr": 1.873, "cost_per_test": 0.072656, "token_totals": { "input_tokens": 62497955, "output_tokens": 4342791, "reasoning_tokens": 3692566, "cache_read_tokens": 56788224, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 86, "avg_turns": 15.2, "avg_query_seconds": 40.2, "median_query_seconds": 11.7, "avg_ttft_seconds": 2.6, "output_time_split": { "reasoning_pct": 90.3, "content_pct": 9.7, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 67.416, "latency": 890.917, "stderr": 0.649, "cost_per_test": 0.448084, "token_totals": { "input_tokens": 43506113, "output_tokens": 2658371, "reasoning_tokens": 2141241, "cache_read_tokens": 36047403, "cache_write_tokens": null }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 16.0, "avg_query_seconds": 37.6, "median_query_seconds": 13.2, "avg_ttft_seconds": 4.1, "output_time_split": { "reasoning_pct": 82.9, "content_pct": 17.1, "tool_call_pct": 0.0 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 67.041, "latency": 683.03, "stderr": 1.35, "cost_per_test": 0.021068, "token_totals": { "input_tokens": 54370624, "output_tokens": 4785981, "reasoning_tokens": 3945320, "cache_read_tokens": 51580757, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 88, "avg_turns": 22.5, "avg_query_seconds": 23.8, "median_query_seconds": 13.4, "avg_ttft_seconds": 1.1, "output_time_split": { "reasoning_pct": 88.7, "content_pct": 11.3, "tool_call_pct": 0.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 67.041, "latency": 784.159, "stderr": 0.375, "cost_per_test": 0.256503, "token_totals": { "input_tokens": 66026998, "output_tokens": 1942653, "reasoning_tokens": null, "cache_read_tokens": 62720352, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 21.2, "avg_query_seconds": 22.1, "median_query_seconds": 10.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 62.921, "latency": 965.737, "stderr": 2.247, "cost_per_test": 0.028403, "token_totals": { "input_tokens": 87337009, "output_tokens": 3334054, "reasoning_tokens": null, "cache_read_tokens": 80932267, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 20.7, "avg_query_seconds": 30.6, "median_query_seconds": 13.5, "avg_ttft_seconds": 1.3, "output_time_split": { "reasoning_pct": 74.2, "content_pct": 25.8, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 61.049, "latency": 935.196, "stderr": 0.375, "cost_per_test": 0.33233, "token_totals": { "input_tokens": 20525184, "output_tokens": 1673824, "reasoning_tokens": 1260207, "cache_read_tokens": 15239680, "cache_write_tokens": null }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 11.5, "avg_query_seconds": 26.9, "median_query_seconds": 8.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 60.674, "latency": 775.239, "stderr": 5.207, "cost_per_test": 0.021658, "token_totals": { "input_tokens": 86304814, "output_tokens": 3550628, "reasoning_tokens": 2582392, "cache_read_tokens": 81263296, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 22.2, "avg_query_seconds": 18.2, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "cursor/composer-2.5": { "accuracy": 58.427, "latency": 0.0, "stderr": 5.254, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 200000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cursor", "harness": "Cursor CLI" }, "alibaba/qwen3.8-27b": { "accuracy": 58.427, "latency": 892.585, "stderr": 5.254, "cost_per_test": 0.677989, "token_totals": { "input_tokens": 95606478, "output_tokens": 4179262, "reasoning_tokens": 3317358, "cache_read_tokens": 0, "cache_write_tokens": null }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 21.6, "avg_query_seconds": 15.4, "median_query_seconds": 6.0, "avg_ttft_seconds": 2.5, "output_time_split": { "reasoning_pct": 82.3, "content_pct": 17.7, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-4-argon": { "accuracy": 57.678, "latency": 841.802, "stderr": 2.456, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5-codex": { "accuracy": 57.303, "latency": 202.469, "stderr": 5.273, "cost_per_test": 0.645217, "token_totals": { "input_tokens": 49121192, "output_tokens": 613091, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Codex" }, "openai/gpt-5.5-factory": { "accuracy": 57.303, "latency": 389.265, "stderr": 5.273, "cost_per_test": 0.751531, "token_totals": { "input_tokens": 39439811, "output_tokens": 744918, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "token_tasks_measured": 89, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Factory" }, "anthropic/claude-sonnet-4-6": { "accuracy": 57.303, "latency": 704.225, "stderr": 0.649, "cost_per_test": 0.572392, "token_totals": { "input_tokens": 52390316, "output_tokens": 1600704, "reasoning_tokens": null, "cache_read_tokens": 49128739, "cache_write_tokens": 3211949 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 19.9, "avg_query_seconds": 15.3, "median_query_seconds": 6.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 57.303, "latency": 866.253, "stderr": 5.273, "cost_per_test": 0.043613, "token_totals": { "input_tokens": 55124187, "output_tokens": 2331687, "reasoning_tokens": 1538667, "cache_read_tokens": 51288960, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 88, "avg_turns": 19.2, "avg_query_seconds": 27.9, "median_query_seconds": 11.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.1": { "accuracy": 56.929, "latency": 940.943, "stderr": 1.35, "cost_per_test": 0.232386, "token_totals": { "input_tokens": 50906500, "output_tokens": 1815010, "reasoning_tokens": 1071648, "cache_read_tokens": 45040256, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 21.4, "avg_query_seconds": 30.3, "median_query_seconds": 15.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 55.056, "latency": 603.689, "stderr": 2.339, "cost_per_test": 0.127848, "token_totals": { "input_tokens": 112525802, "output_tokens": 2949484, "reasoning_tokens": null, "cache_read_tokens": 107994240, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 39.8, "avg_query_seconds": 11.4, "median_query_seconds": 4.6, "avg_ttft_seconds": 2.0, "output_time_split": { "reasoning_pct": 84.3, "content_pct": 15.7, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "tencent/hy4-preview": { "accuracy": 55.056, "latency": 1109.129, "stderr": 3.433, "cost_per_test": 0.140288, "token_totals": { "input_tokens": 58874316, "output_tokens": 2524449, "reasoning_tokens": 1360245, "cache_read_tokens": 54651691, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 86, "avg_turns": 25.1, "avg_query_seconds": 22.5, "median_query_seconds": 9.9, "avg_ttft_seconds": 3.3, "output_time_split": { "reasoning_pct": 62.1, "content_pct": 37.9, "tool_call_pct": 0.0 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 54.682, "latency": 673.036, "stderr": 4.556, "cost_per_test": 0.286739, "token_totals": { "input_tokens": 20716925, "output_tokens": 4555260, "reasoning_tokens": 3735861, "cache_read_tokens": 15580075, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 13.9, "avg_query_seconds": 37.8, "median_query_seconds": 10.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 54.682, "latency": 862.374, "stderr": 1.498, "cost_per_test": 0.199883, "token_totals": { "input_tokens": 41799035, "output_tokens": 3293030, "reasoning_tokens": 2648603, "cache_read_tokens": 39518421, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 19.8, "avg_query_seconds": 34.6, "median_query_seconds": 20.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 53.933, "latency": 376.158, "stderr": 1.297, "cost_per_test": 0.154312, "token_totals": { "input_tokens": 40724231, "output_tokens": 2528658, "reasoning_tokens": 1486431, "cache_read_tokens": 31587323, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 18.1, "avg_query_seconds": 9.6, "median_query_seconds": 3.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 53.558, "latency": 932.358, "stderr": 2.085, "cost_per_test": 0.138449, "token_totals": { "input_tokens": 21929461, "output_tokens": 1568451, "reasoning_tokens": 0, "cache_read_tokens": 18714965, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 88, "avg_turns": 17.0, "avg_query_seconds": 41.9, "median_query_seconds": 15.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 53.558, "latency": 1023.416, "stderr": 0.749, "cost_per_test": 0.205973, "token_totals": { "input_tokens": 64485322, "output_tokens": 2119689, "reasoning_tokens": null, "cache_read_tokens": 53014239, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 88, "avg_turns": 26.1, "avg_query_seconds": 26.9, "median_query_seconds": 9.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 53.184, "latency": 778.109, "stderr": 1.633, "cost_per_test": 0.201902, "token_totals": { "input_tokens": 22196424, "output_tokens": 2290370, "reasoning_tokens": 1654262, "cache_read_tokens": 0, "cache_write_tokens": null }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 15.2, "avg_query_seconds": 33.6, "median_query_seconds": 9.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 52.809, "latency": 654.983, "stderr": 0.649, "cost_per_test": 0.095849, "token_totals": { "input_tokens": 53627914, "output_tokens": 1867100, "reasoning_tokens": 1116365, "cache_read_tokens": 44188800, "cache_write_tokens": null }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 88, "avg_turns": 20.5, "avg_query_seconds": 21.1, "median_query_seconds": 8.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 50.936, "latency": 584.116, "stderr": 2.925, "cost_per_test": null, "token_totals": { "input_tokens": 111289804, "output_tokens": 2704899, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": null }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 88, "avg_turns": 26.9, "avg_query_seconds": 11.8, "median_query_seconds": 5.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 50.187, "latency": 416.383, "stderr": 0.991, "cost_per_test": 0.337052, "token_totals": { "input_tokens": 247145811, "output_tokens": 2141665, "reasoning_tokens": 1125438, "cache_read_tokens": 183334301, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 51.5, "avg_query_seconds": 2.2, "median_query_seconds": 1.5, "avg_ttft_seconds": 1.3, "output_time_split": { "reasoning_pct": 37.9, "content_pct": 62.1, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 50.187, "latency": 593.826, "stderr": 2.456, "cost_per_test": 0.067399, "token_totals": { "input_tokens": 210224480, "output_tokens": 3744590, "reasoning_tokens": null, "cache_read_tokens": 194621013, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 83, "avg_turns": 40.7, "avg_query_seconds": 4.5, "median_query_seconds": 2.5, "avg_ttft_seconds": 1.6, "output_time_split": { "reasoning_pct": 58.4, "content_pct": 41.6, "tool_call_pct": 0.0 } }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 50.187, "latency": 833.528, "stderr": 1.35, "cost_per_test": 0.043393, "token_totals": { "input_tokens": 136334405, "output_tokens": 7151114, "reasoning_tokens": null, "cache_read_tokens": 118584064, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 87, "avg_turns": 24.9, "avg_query_seconds": 26.6, "median_query_seconds": 10.0, "avg_ttft_seconds": 1.8, "output_time_split": { "reasoning_pct": 88.0, "content_pct": 12.0, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 50.187, "latency": 1078.158, "stderr": 1.498, "cost_per_test": 0.239191, "token_totals": { "input_tokens": 36128386, "output_tokens": 3648724, "reasoning_tokens": 2947326, "cache_read_tokens": 34026923, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 87, "avg_turns": 21.4, "avg_query_seconds": 43.8, "median_query_seconds": 19.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 48.689, "latency": 812.638, "stderr": 3.066, "cost_per_test": 0.066012, "token_totals": { "input_tokens": 90437019, "output_tokens": 1897074, "reasoning_tokens": null, "cache_read_tokens": 87265378, "cache_write_tokens": 388109 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 31.3, "avg_query_seconds": 12.6, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 47.566, "latency": 771.551, "stderr": 0.375, "cost_per_test": 0.630691, "token_totals": { "input_tokens": 226227937, "output_tokens": 3179128, "reasoning_tokens": null, "cache_read_tokens": 220634923, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 56.5, "avg_query_seconds": 9.7, "median_query_seconds": 4.1, "avg_ttft_seconds": 1.8, "output_time_split": { "reasoning_pct": 78.1, "content_pct": 21.9, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 44.195, "latency": 305.726, "stderr": 0.991, "cost_per_test": 0.407362, "token_totals": { "input_tokens": 28275678, "output_tokens": 2268536, "reasoning_tokens": 1952049, "cache_read_tokens": 18837397, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 12.3, "avg_query_seconds": 17.9, "median_query_seconds": 11.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 43.82, "latency": 649.489, "stderr": 1.946, "cost_per_test": 0.363822, "token_totals": { "input_tokens": 84235092, "output_tokens": 4076150, "reasoning_tokens": null, "cache_read_tokens": 80999717, "cache_write_tokens": 2656167 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 26.0, "avg_query_seconds": 14.1, "median_query_seconds": 5.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 41.948, "latency": 470.713, "stderr": 2.622, "cost_per_test": 0.982203, "token_totals": { "input_tokens": 232260469, "output_tokens": 2597032, "reasoning_tokens": 1420634, "cache_read_tokens": 220796992, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 36.2, "avg_query_seconds": 6.9, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 41.948, "latency": 1051.079, "stderr": 0.749, "cost_per_test": 0.085844, "token_totals": { "input_tokens": 34513305, "output_tokens": 1049061, "reasoning_tokens": 355219, "cache_read_tokens": 32430174, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 88, "avg_turns": 22.0, "avg_query_seconds": 33.9, "median_query_seconds": 23.0, "avg_ttft_seconds": 19.3, "output_time_split": { "reasoning_pct": 35.2, "content_pct": 64.8, "tool_call_pct": 0.0 } }, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 41.573, "latency": 757.485, "stderr": 2.973, "cost_per_test": 0.072321, "token_totals": { "input_tokens": 30849406, "output_tokens": 3569635, "reasoning_tokens": 2418488, "cache_read_tokens": 23307435, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 16.4, "avg_query_seconds": 20.9, "median_query_seconds": 15.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 38.951, "latency": 989.169, "stderr": 2.701, "cost_per_test": 1.190984, "token_totals": { "input_tokens": 55841015, "output_tokens": 2964803, "reasoning_tokens": null, "cache_read_tokens": 0, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 20.7, "avg_query_seconds": 14.9, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 34.457, "latency": 688.923, "stderr": 0.749, "cost_per_test": 0.214987, "token_totals": { "input_tokens": 158620249, "output_tokens": 2324434, "reasoning_tokens": 820396, "cache_read_tokens": 79630802, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 87, "avg_turns": 41.2, "avg_query_seconds": 3.0, "median_query_seconds": 1.9, "avg_ttft_seconds": 2.5, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 34.082, "latency": 278.342, "stderr": 0.991, "cost_per_test": 0.084894, "token_totals": { "input_tokens": 36882464, "output_tokens": 2980609, "reasoning_tokens": 2461322, "cache_read_tokens": 27270935, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 13.4, "avg_query_seconds": 9.3, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 34.082, "latency": 1209.608, "stderr": 1.633, "cost_per_test": 0.300575, "token_totals": { "input_tokens": 223144546, "output_tokens": 2120066, "reasoning_tokens": 495317, "cache_read_tokens": 187257533, "cache_write_tokens": null }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 49.5, "avg_query_seconds": 17.3, "median_query_seconds": 11.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 25.843, "latency": 1218.578, "stderr": 2.973, "cost_per_test": 0.253714, "token_totals": { "input_tokens": 291488884, "output_tokens": 2546703, "reasoning_tokens": 621785, "cache_read_tokens": 141552645, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 67.0, "avg_query_seconds": 13.4, "median_query_seconds": 8.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "cohere/command-a-plus-05-2026": { "accuracy": 17.603, "latency": 0.0, "stderr": 0.375, "cost_per_test": null, "token_totals": { "input_tokens": 72409970, "output_tokens": 3459251, "reasoning_tokens": 2557941, "cache_read_tokens": 0, "cache_write_tokens": null }, "token_tasks_measured": 89, "turn_metrics": null, "temperature": 0.9, "top_p": 0.95, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 10.861, "latency": 1028.327, "stderr": 2.622, "cost_per_test": 0.014296, "token_totals": { "input_tokens": 73183797, "output_tokens": 2194003, "reasoning_tokens": null, "cache_read_tokens": 70641781, "cache_write_tokens": 0 }, "token_tasks_measured": 89, "turn_metrics": { "tasks_measured": 89, "avg_turns": 22.4, "avg_query_seconds": 11.3, "median_query_seconds": 1.3, "avg_ttft_seconds": 1.4, "output_time_split": { "reasoning_pct": 64.9, "content_pct": 35.1, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "easy": { "cursor/composer-2.5": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 200000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cursor", "harness": "Cursor CLI" }, "openai/gpt-6-astra": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8-claude-code": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "Claude Code" }, "google/gemini-3.5-flash": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.2": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 48000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 100.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.5": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "thinkingmachines/inkling": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.7": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 91.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 83.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 83.333, "latency": null, "stderr": 16.667, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 83.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 83.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 83.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 83.333, "latency": null, "stderr": 16.667, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 83.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 83.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 83.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.1": { "accuracy": 83.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 83.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.5-codex": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Codex" }, "google/gemini-3-flash-preview": { "accuracy": 75.0, "latency": null, "stderr": 14.434, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5-factory": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Factory" }, "grok/grok-4.3": { "accuracy": 75.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 75.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 75.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 75.0, "latency": null, "stderr": 14.434, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 75.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 75.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 75.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-4-argon": { "accuracy": 75.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 75.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 75.0, "latency": null, "stderr": 14.434, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 75.0, "latency": null, "stderr": 25.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 75.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 75.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 75.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 75.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "inception/mercury-2.5": { "accuracy": 66.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 66.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 66.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 66.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 66.667, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 58.333, "latency": null, "stderr": 16.667, "cost_per_test": null, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 58.333, "latency": null, "stderr": 8.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "cohere/command-a-plus-05-2026": { "accuracy": 50.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.9, "top_p": 0.95, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 50.0, "latency": null, "stderr": 28.868, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 41.667, "latency": null, "stderr": 16.667, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "medium": { "anthropic/claude-opus-5-5": { "accuracy": 90.303, "latency": null, "stderr": 1.603, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 86.061, "latency": null, "stderr": 0.606, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 86.061, "latency": null, "stderr": 2.185, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 86.061, "latency": null, "stderr": 1.212, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 85.455, "latency": null, "stderr": 1.05, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 84.848, "latency": null, "stderr": 0.606, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 84.848, "latency": null, "stderr": 0.606, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 84.242, "latency": null, "stderr": 1.603, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 83.636, "latency": null, "stderr": 1.05, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 83.636, "latency": null, "stderr": 1.818, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 83.636, "latency": null, "stderr": 1.818, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.6": { "accuracy": 83.636, "latency": null, "stderr": 1.818, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 83.636, "latency": null, "stderr": 1.05, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 83.03, "latency": null, "stderr": 1.212, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 82.424, "latency": null, "stderr": 0.606, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 81.818, "latency": null, "stderr": 2.777, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 81.212, "latency": null, "stderr": 0.606, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 81.212, "latency": null, "stderr": 1.212, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 80.0, "latency": null, "stderr": 1.05, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3": { "accuracy": 80.0, "latency": null, "stderr": 1.818, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 79.394, "latency": null, "stderr": 1.603, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 79.394, "latency": null, "stderr": 1.603, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 78.182, "latency": null, "stderr": 1.05, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 78.182, "latency": null, "stderr": 1.05, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 78.182, "latency": null, "stderr": 1.818, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 76.97, "latency": null, "stderr": 1.603, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 76.97, "latency": null, "stderr": 3.207, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 76.364, "latency": null, "stderr": 1.05, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 75.758, "latency": null, "stderr": 2.424, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 75.152, "latency": null, "stderr": 1.212, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 75.152, "latency": null, "stderr": 1.212, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 75.152, "latency": null, "stderr": 1.603, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.2": { "accuracy": 74.545, "latency": null, "stderr": 1.05, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 48000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 74.545, "latency": null, "stderr": 1.05, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 73.939, "latency": null, "stderr": 3.207, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.5": { "accuracy": 72.121, "latency": null, "stderr": 5.781, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-8-claude-code": { "accuracy": 70.909, "latency": null, "stderr": 6.181, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "Claude Code" }, "xiaomi/mimo-v2.5": { "accuracy": 70.909, "latency": null, "stderr": 6.181, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 69.697, "latency": null, "stderr": 4.242, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 68.485, "latency": null, "stderr": 1.212, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 67.273, "latency": null, "stderr": 1.818, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 67.273, "latency": null, "stderr": 2.777, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 67.273, "latency": null, "stderr": 6.385, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 66.667, "latency": null, "stderr": 2.185, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 66.667, "latency": null, "stderr": 1.603, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 66.667, "latency": null, "stderr": 3.687, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "cursor/composer-2.5": { "accuracy": 65.455, "latency": null, "stderr": 6.471, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 200000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cursor", "harness": "Cursor CLI" }, "openai/gpt-5.5-factory": { "accuracy": 65.455, "latency": null, "stderr": 6.471, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Factory" }, "xiaomi/mimo-v2.5-pro": { "accuracy": 65.455, "latency": null, "stderr": 6.471, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.1": { "accuracy": 65.455, "latency": null, "stderr": 1.05, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 64.242, "latency": null, "stderr": 3.687, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 64.242, "latency": null, "stderr": 1.603, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.5-codex": { "accuracy": 63.636, "latency": null, "stderr": 6.546, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Codex" }, "alibaba/qwen3.6-plus": { "accuracy": 63.636, "latency": null, "stderr": 2.099, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 63.03, "latency": null, "stderr": 2.642, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "tencent/hy4-preview": { "accuracy": 63.03, "latency": null, "stderr": 3.207, "cost_per_test": null, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 62.424, "latency": null, "stderr": 2.424, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 62.424, "latency": null, "stderr": 2.424, "cost_per_test": null, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 60.0, "latency": null, "stderr": 2.777, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 59.394, "latency": null, "stderr": 1.603, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 59.394, "latency": null, "stderr": 1.603, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-4-argon": { "accuracy": 58.788, "latency": null, "stderr": 2.185, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling": { "accuracy": 57.576, "latency": null, "stderr": 1.212, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 57.576, "latency": null, "stderr": 2.185, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 56.364, "latency": null, "stderr": 1.05, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.3": { "accuracy": 54.545, "latency": null, "stderr": 3.785, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 52.727, "latency": null, "stderr": 3.785, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 52.121, "latency": null, "stderr": 1.603, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 49.697, "latency": null, "stderr": 4.242, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 47.273, "latency": null, "stderr": 1.05, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 44.242, "latency": null, "stderr": 0.606, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-m.1": { "accuracy": 42.424, "latency": null, "stderr": 2.642, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 41.818, "latency": null, "stderr": 1.05, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 31.515, "latency": null, "stderr": 3.207, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "cohere/command-a-plus-05-2026": { "accuracy": 23.03, "latency": null, "stderr": 0.606, "cost_per_test": null, "temperature": 0.9, "top_p": 0.95, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 13.939, "latency": null, "stderr": 3.374, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "hard": { "openai/gpt-6-astra": { "accuracy": 87.778, "latency": null, "stderr": 2.222, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 84.444, "latency": null, "stderr": 2.94, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 84.444, "latency": null, "stderr": 1.111, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 82.222, "latency": null, "stderr": 1.111, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 81.111, "latency": null, "stderr": 2.94, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 78.889, "latency": null, "stderr": 2.94, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 77.778, "latency": null, "stderr": 2.94, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 76.667, "latency": null, "stderr": 5.092, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 74.444, "latency": null, "stderr": 6.186, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 72.222, "latency": null, "stderr": 2.222, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 72.222, "latency": null, "stderr": 1.111, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 71.111, "latency": null, "stderr": 4.006, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 70.0, "latency": null, "stderr": 1.925, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 70.0, "latency": null, "stderr": 3.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 68.889, "latency": null, "stderr": 2.94, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.7": { "accuracy": 67.778, "latency": null, "stderr": 2.94, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 66.667, "latency": null, "stderr": 1.925, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 65.556, "latency": null, "stderr": 2.94, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 64.444, "latency": null, "stderr": 2.222, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 64.444, "latency": null, "stderr": 2.222, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8-claude-code": { "accuracy": 63.333, "latency": null, "stderr": 8.949, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": "Claude Code" }, "xiaomi/mimo-v2.6-pro": { "accuracy": 63.333, "latency": null, "stderr": 3.333, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 61.111, "latency": null, "stderr": 1.111, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 60.0, "latency": null, "stderr": 1.925, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 60.0, "latency": null, "stderr": 1.925, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 60.0, "latency": null, "stderr": 1.925, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 57.778, "latency": null, "stderr": 2.222, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 56.667, "latency": null, "stderr": 1.925, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.5": { "accuracy": 56.667, "latency": null, "stderr": 1.925, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 53.333, "latency": null, "stderr": 3.333, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 53.333, "latency": null, "stderr": 3.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 53.333, "latency": null, "stderr": 1.925, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 52.222, "latency": null, "stderr": 2.94, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.2": { "accuracy": 51.111, "latency": null, "stderr": 2.222, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 48000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 51.111, "latency": null, "stderr": 1.111, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 45.556, "latency": null, "stderr": 4.006, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 44.444, "latency": null, "stderr": 1.111, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.5-codex": { "accuracy": 43.333, "latency": null, "stderr": 9.202, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Codex" }, "alibaba/qwen3.8-27b": { "accuracy": 43.333, "latency": null, "stderr": 9.202, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 41.111, "latency": null, "stderr": 2.222, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "cursor/composer-2.5": { "accuracy": 40.0, "latency": null, "stderr": 9.097, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 200000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cursor", "harness": "Cursor CLI" }, "google/gemini-3-flash-preview": { "accuracy": 40.0, "latency": null, "stderr": 3.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5-factory": { "accuracy": 40.0, "latency": null, "stderr": 9.097, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": "Factory" }, "anthropic/claude-opus-4-7": { "accuracy": 40.0, "latency": null, "stderr": 5.092, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 40.0, "latency": null, "stderr": 9.097, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 40.0, "latency": null, "stderr": 3.849, "cost_per_test": null, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 38.889, "latency": null, "stderr": 2.94, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.1": { "accuracy": 37.778, "latency": null, "stderr": 1.111, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 36.667, "latency": null, "stderr": 8.949, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 34.444, "latency": null, "stderr": 6.759, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 34.444, "latency": null, "stderr": 2.94, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 31.111, "latency": null, "stderr": 1.111, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 31.111, "latency": null, "stderr": 1.111, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 31.111, "latency": null, "stderr": 1.111, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 31.111, "latency": null, "stderr": 2.222, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 30.0, "latency": null, "stderr": 1.925, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 28.889, "latency": null, "stderr": 4.444, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 28.889, "latency": null, "stderr": 2.94, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 25.556, "latency": null, "stderr": 4.843, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 25.556, "latency": null, "stderr": 2.222, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 24.444, "latency": null, "stderr": 4.006, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 23.333, "latency": null, "stderr": 1.925, "cost_per_test": null, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 23.333, "latency": null, "stderr": 3.849, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 23.333, "latency": null, "stderr": 1.925, "cost_per_test": null, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 22.222, "latency": null, "stderr": 1.111, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 18.889, "latency": null, "stderr": 5.879, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 18.889, "latency": null, "stderr": 4.006, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 16.667, "latency": null, "stderr": 1.925, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 15.556, "latency": null, "stderr": 1.111, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 15.556, "latency": null, "stderr": 2.222, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.3": { "accuracy": 14.444, "latency": null, "stderr": 2.222, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 13.333, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "inception/mercury-2.5": { "accuracy": 12.222, "latency": null, "stderr": 2.222, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 11.111, "latency": null, "stderr": 2.94, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "cohere/command-a-plus-05-2026": { "accuracy": 3.333, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 0.9, "top_p": 0.95, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 1.111, "latency": null, "stderr": 1.111, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } } } }