{ "metadata": { "benchmark": "Terminal-Bench 1.0", "slug": "terminal-bench", "description": "State-of-the-art set of difficult terminal-based tasks", "benchmark_id": "terminal_bench", "family": "terminal_bench", "version": "1.0", "updated": "2026-01-12", "dataset_type": "public", "industry": "coding", "tasks": { "overall": "Overall", "easy": "Easy", "medium": "Medium", "hard": "Hard" }, "models": [ "ai21labs/jamba-large-1.7", "alibaba/qwen3-max", "alibaba/qwen3-max-preview", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929-thinking", "cohere/command-a-03-2025", "fireworks/deepseek-r1", "fireworks/deepseek-v3p1", "fireworks/deepseek-v3p2", "fireworks/deepseek-v3p2-thinking", "fireworks/gpt-oss-120b", "fireworks/kimi-k2-instruct-0905", "fireworks/llama4-maverick-instruct-basic", "google/gemini-2.5-flash-preview-09-2025", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-flash-thinking", "google/gemini-2.5-pro", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-code-fast-1", "kimi/kimi-k2-thinking", "minimax/MiniMax-M2.1", "mistralai/devstral-2512", "mistralai/labs-devstral-small-2512", "mistralai/magistral-medium-2509", "mistralai/magistral-small-2509", "mistralai/mistral-large-2512", "openai/gpt-4.1-2025-04-14", "openai/gpt-5-2025-08-07", "openai/gpt-5-codex", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.1-codex", "openai/gpt-5.1-codex-max", "openai/gpt-5.2-2025-12-11", "together/moonshotai/Kimi-K2-Instruct", "zai/glm-4.5", "zai/glm-4.6", "zai/glm-4.7" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "platform", "mode": "agentic", "archived": false, "total_models": 47 }, "tasks": { "overall": { "openai/gpt-5.2-2025-12-11": { "accuracy": 63.75, "latency": 1429.571, "stderr": 5.375, "cost_per_test": 1.395387, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 61.25, "latency": 525.778, "stderr": 5.447, "cost_per_test": 0.660783, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3-flash-preview": { "accuracy": 60.0, "latency": 118.432, "stderr": 5.477, "cost_per_test": 0.125536, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5-codex": { "accuracy": 58.75, "latency": 663.824, "stderr": 5.504, "cost_per_test": 1.380093, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 57.5, "latency": 309.102, "stderr": 5.527, "cost_per_test": 0.960696, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.1-codex": { "accuracy": 57.5, "latency": 946.453, "stderr": 5.527, "cost_per_test": 0.121214, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 56.25, "latency": 177.102, "stderr": 5.546, "cost_per_test": 0.744198, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.1-codex-max": { "accuracy": 53.75, "latency": 711.822, "stderr": 5.574, "cost_per_test": 1.018534, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3-pro-preview": { "accuracy": 51.25, "latency": 508.608, "stderr": 5.588, "cost_per_test": 0.318077, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 50.0, "latency": 349.265, "stderr": 5.59, "cost_per_test": 0.259805, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3p2": { "accuracy": 50.0, "latency": 399.137, "stderr": 5.59, "cost_per_test": 0.024996, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "zai/glm-4.7": { "accuracy": 50.0, "latency": 562.429, "stderr": 5.59, "cost_per_test": 0.223346, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "openai/gpt-5-2025-08-07": { "accuracy": 48.75, "latency": 957.628, "stderr": 5.588, "cost_per_test": 0.100635, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 47.5, "latency": 685.951, "stderr": 5.583, "cost_per_test": 0.10176, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 45.0, "latency": 717.326, "stderr": 5.562, "cost_per_test": 2.017755, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/devstral-2512": { "accuracy": 43.75, "latency": 488.55, "stderr": 5.546, "cost_per_test": 0.424819, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "zai/glm-4.6": { "accuracy": 42.5, "latency": 135.723, "stderr": 5.527, "cost_per_test": 0.285842, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "zai/glm-4.5": { "accuracy": 41.25, "latency": 169.409, "stderr": 5.504, "cost_per_test": 0.197232, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "google/gemini-2.5-pro": { "accuracy": 41.25, "latency": 352.476, "stderr": 5.504, "cost_per_test": 0.664031, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "minimax/MiniMax-M2.1": { "accuracy": 41.25, "latency": 442.254, "stderr": 5.504, "cost_per_test": 0.062918, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "fireworks/deepseek-v3p1": { "accuracy": 41.25, "latency": 641.23, "stderr": 5.504, "cost_per_test": 0.391033, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "kimi/kimi-k2-thinking": { "accuracy": 40.0, "latency": 133.903, "stderr": 5.477, "cost_per_test": 0.33518, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 40.0, "latency": 133.903, "stderr": 5.477, "cost_per_test": 0.33518, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "mistralai/labs-devstral-small-2512": { "accuracy": 40.0, "latency": 285.171, "stderr": 5.477, "cost_per_test": 0.135297, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 40.0, "latency": 341.987, "stderr": 5.477, "cost_per_test": 0.047832, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-4-0709": { "accuracy": 38.75, "latency": 902.175, "stderr": 5.447, "cost_per_test": 7.31463, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/kimi-k2-instruct-0905": { "accuracy": 37.5, "latency": 149.814, "stderr": 5.413, "cost_per_test": 0.433024, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "alibaba/qwen3-max-preview": { "accuracy": 36.25, "latency": 172.116, "stderr": 5.375, "cost_per_test": 0.715376, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "alibaba/qwen3-max": { "accuracy": 36.25, "latency": 297.376, "stderr": 5.375, "cost_per_test": 0.480579, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 33.75, "latency": 325.463, "stderr": 5.287, "cost_per_test": 1.292226, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 31.25, "latency": 79.464, "stderr": 5.182, "cost_per_test": 0.134487, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 30.0, "latency": 1485.244, "stderr": 5.124, "cost_per_test": 0.366108, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 28.75, "latency": 238.437, "stderr": 5.06, "cost_per_test": 0.009636, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/magistral-medium-2509": { "accuracy": 28.75, "latency": 746.79, "stderr": 5.06, "cost_per_test": 1.234078, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-4-fast-reasoning": { "accuracy": 27.5, "latency": 134.967, "stderr": 4.992, "cost_per_test": 0.188369, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 26.25, "latency": 95.3, "stderr": 4.919, "cost_per_test": 0.132934, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/gpt-oss-120b": { "accuracy": 22.5, "latency": 53.75, "stderr": 4.669, "cost_per_test": 0.038818, "temperature": null, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 21.25, "latency": 133.157, "stderr": 4.574, "cost_per_test": 0.02719, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-large-2512": { "accuracy": 21.25, "latency": 158.436, "stderr": 4.574, "cost_per_test": 0.057283, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "google/gemini-2.5-flash-thinking": { "accuracy": 21.25, "latency": 244.663, "stderr": 4.574, "cost_per_test": 0.447777, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-code-fast-1": { "accuracy": 20.0, "latency": 85.132, "stderr": 4.472, "cost_per_test": 0.237938, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 18.75, "latency": 65.358, "stderr": 4.364, "cost_per_test": 0.311069, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 17.5, "latency": 55.424, "stderr": 4.248, "cost_per_test": 0.03501, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "mistralai/magistral-small-2509": { "accuracy": 15.0, "latency": 607.926, "stderr": 3.992, "cost_per_test": 0.22067, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/deepseek-r1": { "accuracy": 13.75, "latency": 145.869, "stderr": 3.85, "cost_per_test": 0.359797, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "cohere/command-a-03-2025": { "accuracy": 6.25, "latency": 287.604, "stderr": 2.706, "cost_per_test": 2.5047, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "ai21labs/jamba-large-1.7": { "accuracy": 6.25, "latency": 578.51, "stderr": 2.706, "cost_per_test": 4.016933, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } }, "easy": { "openai/gpt-5.2-2025-12-11": { "accuracy": 100.0, "latency": 447.42, "stderr": 0.0, "cost_per_test": 0.531567, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3-pro-preview": { "accuracy": 91.667, "latency": 273.056, "stderr": 7.979, "cost_per_test": 0.141045, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5-codex": { "accuracy": 91.667, "latency": 282.108, "stderr": 7.979, "cost_per_test": 0.734236, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.1-codex": { "accuracy": 91.667, "latency": 459.874, "stderr": 7.979, "cost_per_test": 0.069293, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 83.333, "latency": 36.004, "stderr": 10.758, "cost_per_test": 0.035971, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3-flash-preview": { "accuracy": 83.333, "latency": 65.318, "stderr": 10.758, "cost_per_test": 0.072666, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/devstral-2512": { "accuracy": 83.333, "latency": 78.666, "stderr": 10.758, "cost_per_test": 0.097929, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "minimax/MiniMax-M2.1": { "accuracy": 83.333, "latency": 144.162, "stderr": 10.758, "cost_per_test": 0.019861, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "google/gemini-2.5-pro": { "accuracy": 83.333, "latency": 204.504, "stderr": 10.758, "cost_per_test": 0.366538, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 83.333, "latency": 212.927, "stderr": 10.758, "cost_per_test": 0.720249, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 83.333, "latency": 314.122, "stderr": 10.758, "cost_per_test": 0.347151, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-4-0709": { "accuracy": 83.333, "latency": 317.016, "stderr": 10.758, "cost_per_test": 6.078042, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 83.333, "latency": 370.297, "stderr": 10.758, "cost_per_test": 0.058654, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 75.0, "latency": 52.27, "stderr": 12.5, "cost_per_test": 0.1026, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/kimi-k2-instruct-0905": { "accuracy": 75.0, "latency": 64.894, "stderr": 12.5, "cost_per_test": 0.159374, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "zai/glm-4.6": { "accuracy": 75.0, "latency": 78.591, "stderr": 12.5, "cost_per_test": 0.168286, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 75.0, "latency": 87.407, "stderr": 12.5, "cost_per_test": 0.141256, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 75.0, "latency": 87.827, "stderr": 12.5, "cost_per_test": 0.508694, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 75.0, "latency": 97.077, "stderr": 12.5, "cost_per_test": 0.103818, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 75.0, "latency": 228.124, "stderr": 12.5, "cost_per_test": 0.782573, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.1-codex-max": { "accuracy": 75.0, "latency": 360.118, "stderr": 12.5, "cost_per_test": 0.478942, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3-max-preview": { "accuracy": 66.667, "latency": 37.332, "stderr": 13.608, "cost_per_test": 0.043625, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "kimi/kimi-k2-thinking": { "accuracy": 66.667, "latency": 64.271, "stderr": 13.608, "cost_per_test": 0.124191, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 66.667, "latency": 64.271, "stderr": 13.608, "cost_per_test": 0.124191, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "fireworks/deepseek-r1": { "accuracy": 66.667, "latency": 69.914, "stderr": 13.608, "cost_per_test": 0.098143, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "zai/glm-4.5": { "accuracy": 66.667, "latency": 80.184, "stderr": 13.608, "cost_per_test": 0.066312, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 66.667, "latency": 116.214, "stderr": 13.608, "cost_per_test": 0.002913, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "alibaba/qwen3-max": { "accuracy": 66.667, "latency": 162.446, "stderr": 13.608, "cost_per_test": 0.230463, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "zai/glm-4.7": { "accuracy": 66.667, "latency": 212.903, "stderr": 13.608, "cost_per_test": 0.085327, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "fireworks/deepseek-v3p2": { "accuracy": 66.667, "latency": 257.373, "stderr": 13.608, "cost_per_test": 0.016, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "mistralai/magistral-medium-2509": { "accuracy": 66.667, "latency": 333.004, "stderr": 13.608, "cost_per_test": 0.219626, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-5-2025-08-07": { "accuracy": 66.667, "latency": 485.197, "stderr": 13.608, "cost_per_test": 0.046931, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/labs-devstral-small-2512": { "accuracy": 58.333, "latency": 57.448, "stderr": 14.232, "cost_per_test": 0.04099, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 58.333, "latency": 157.026, "stderr": 14.232, "cost_per_test": 0.018635, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "fireworks/deepseek-v3p1": { "accuracy": 58.333, "latency": 360.512, "stderr": 14.232, "cost_per_test": 0.101422, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 58.333, "latency": 1660.632, "stderr": 14.232, "cost_per_test": 0.427128, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/gpt-oss-120b": { "accuracy": 50.0, "latency": 22.368, "stderr": 14.434, "cost_per_test": 0.008486, "temperature": null, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 50.0, "latency": 25.286, "stderr": 14.434, "cost_per_test": 0.007615, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-4-fast-reasoning": { "accuracy": 50.0, "latency": 51.416, "stderr": 14.434, "cost_per_test": 0.058789, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-large-2512": { "accuracy": 50.0, "latency": 64.666, "stderr": 14.434, "cost_per_test": 0.019452, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 41.667, "latency": 22.117, "stderr": 14.232, "cost_per_test": 0.101459, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 41.667, "latency": 41.63, "stderr": 14.232, "cost_per_test": 0.012043, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-thinking": { "accuracy": 41.667, "latency": 52.831, "stderr": 14.232, "cost_per_test": 0.049553, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/magistral-small-2509": { "accuracy": 41.667, "latency": 317.63, "stderr": 14.232, "cost_per_test": 0.120677, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-code-fast-1": { "accuracy": 33.333, "latency": 29.349, "stderr": 13.608, "cost_per_test": 0.061858, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "ai21labs/jamba-large-1.7": { "accuracy": 33.333, "latency": 282.315, "stderr": 13.608, "cost_per_test": 2.166129, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "cohere/command-a-03-2025": { "accuracy": 16.667, "latency": 32.912, "stderr": 10.758, "cost_per_test": 0.045615, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" } }, "medium": { "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 63.636, "latency": 305.842, "stderr": 7.252, "cost_per_test": 0.385998, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 63.636, "latency": 941.926, "stderr": 7.252, "cost_per_test": 1.021821, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.1-codex-max": { "accuracy": 61.364, "latency": 369.539, "stderr": 7.34, "cost_per_test": 0.481731, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5-codex": { "accuracy": 61.364, "latency": 372.793, "stderr": 7.34, "cost_per_test": 0.712311, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.1-codex": { "accuracy": 59.091, "latency": 616.007, "stderr": 7.412, "cost_per_test": 0.091183, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3-flash-preview": { "accuracy": 56.818, "latency": 109.433, "stderr": 7.467, "cost_per_test": 0.105281, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 56.818, "latency": 128.365, "stderr": 7.467, "cost_per_test": 0.459022, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 56.818, "latency": 185.954, "stderr": 7.467, "cost_per_test": 0.510967, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5-2025-08-07": { "accuracy": 56.818, "latency": 944.164, "stderr": 7.467, "cost_per_test": 0.110508, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 52.273, "latency": 229.623, "stderr": 7.53, "cost_per_test": 0.177241, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3p2": { "accuracy": 52.273, "latency": 357.895, "stderr": 7.53, "cost_per_test": 0.021038, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "zai/glm-4.7": { "accuracy": 52.273, "latency": 373.273, "stderr": 7.53, "cost_per_test": 0.122355, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "google/gemini-3-pro-preview": { "accuracy": 52.273, "latency": 430.838, "stderr": 7.53, "cost_per_test": 0.257726, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 45.454, "latency": 337.07, "stderr": 7.507, "cost_per_test": 0.046125, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 45.454, "latency": 594.689, "stderr": 7.507, "cost_per_test": 0.082609, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2-thinking": { "accuracy": 43.182, "latency": 115.346, "stderr": 7.467, "cost_per_test": 0.259957, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 43.182, "latency": 115.346, "stderr": 7.467, "cost_per_test": 0.259957, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "mistralai/labs-devstral-small-2512": { "accuracy": 43.182, "latency": 240.558, "stderr": 7.467, "cost_per_test": 0.09684, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 43.182, "latency": 609.155, "stderr": 7.467, "cost_per_test": 1.751723, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-4.6": { "accuracy": 40.909, "latency": 119.951, "stderr": 7.412, "cost_per_test": 0.210667, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "minimax/MiniMax-M2.1": { "accuracy": 40.909, "latency": 331.361, "stderr": 7.412, "cost_per_test": 0.049873, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "fireworks/deepseek-v3p1": { "accuracy": 40.909, "latency": 606.496, "stderr": 7.412, "cost_per_test": 0.358373, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "mistralai/devstral-2512": { "accuracy": 40.909, "latency": 659.682, "stderr": 7.412, "cost_per_test": 0.381742, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "alibaba/qwen3-max-preview": { "accuracy": 38.636, "latency": 149.148, "stderr": 7.34, "cost_per_test": 0.562713, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "zai/glm-4.5": { "accuracy": 38.636, "latency": 152.327, "stderr": 7.34, "cost_per_test": 0.162682, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "alibaba/qwen3-max": { "accuracy": 38.636, "latency": 272.89, "stderr": 7.34, "cost_per_test": 0.461304, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "fireworks/kimi-k2-instruct-0905": { "accuracy": 36.364, "latency": 146.28, "stderr": 7.252, "cost_per_test": 0.417926, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-4-0709": { "accuracy": 36.364, "latency": 696.792, "stderr": 7.252, "cost_per_test": 5.436423, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.5-pro": { "accuracy": 34.091, "latency": 246.48, "stderr": 7.146, "cost_per_test": 0.395039, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 34.091, "latency": 384.844, "stderr": 7.146, "cost_per_test": 1.290994, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-fast-reasoning": { "accuracy": 29.546, "latency": 91.921, "stderr": 6.878, "cost_per_test": 0.153257, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 29.546, "latency": 924.141, "stderr": 6.878, "cost_per_test": 0.170694, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 27.273, "latency": 78.561, "stderr": 6.714, "cost_per_test": 0.131812, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/magistral-medium-2509": { "accuracy": 27.273, "latency": 775.463, "stderr": 6.714, "cost_per_test": 1.474164, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "grok/grok-code-fast-1": { "accuracy": 22.727, "latency": 64.888, "stderr": 6.318, "cost_per_test": 0.151074, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/gpt-oss-120b": { "accuracy": 22.727, "latency": 65.798, "stderr": 6.318, "cost_per_test": 0.05647, "temperature": null, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 22.727, "latency": 75.053, "stderr": 6.318, "cost_per_test": 0.3405, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 22.727, "latency": 189.818, "stderr": 6.318, "cost_per_test": 0.006506, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 20.454, "latency": 73.342, "stderr": 6.081, "cost_per_test": 0.096337, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 20.454, "latency": 107.804, "stderr": 6.081, "cost_per_test": 0.025919, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-large-2512": { "accuracy": 20.454, "latency": 157.168, "stderr": 6.081, "cost_per_test": 0.056686, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "google/gemini-2.5-flash-thinking": { "accuracy": 18.182, "latency": 207.651, "stderr": 5.815, "cost_per_test": 0.365229, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 15.909, "latency": 54.342, "stderr": 5.514, "cost_per_test": 0.032137, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "mistralai/magistral-small-2509": { "accuracy": 11.364, "latency": 724.28, "stderr": 4.784, "cost_per_test": 0.289942, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/deepseek-r1": { "accuracy": 6.818, "latency": 152.401, "stderr": 3.8, "cost_per_test": 0.447952, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "cohere/command-a-03-2025": { "accuracy": 4.546, "latency": 236.998, "stderr": 3.14, "cost_per_test": 2.522485, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "ai21labs/jamba-large-1.7": { "accuracy": 2.273, "latency": 494.991, "stderr": 2.247, "cost_per_test": 3.24137, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } }, "hard": { "google/gemini-3-flash-preview": { "accuracy": 54.167, "latency": 164.992, "stderr": 10.171, "cost_per_test": 0.193964, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 45.833, "latency": 309.058, "stderr": 10.171, "cost_per_test": 1.372891, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 45.833, "latency": 577.83, "stderr": 10.171, "cost_per_test": 1.886684, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 45.833, "latency": 1034.821, "stderr": 10.171, "cost_per_test": 1.321373, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 45.833, "latency": 2814.661, "stderr": 10.171, "cost_per_test": 2.512167, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/deepseek-v3p2": { "accuracy": 37.5, "latency": 551.998, "stderr": 9.882, "cost_per_test": 0.037263, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "zai/glm-4.7": { "accuracy": 37.5, "latency": 1076.097, "stderr": 9.882, "cost_per_test": 0.473297, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "openai/gpt-5-codex": { "accuracy": 37.5, "latency": 1388.238, "stderr": 9.882, "cost_per_test": 2.927289, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.1-codex": { "accuracy": 37.5, "latency": 1795.561, "stderr": 9.882, "cost_per_test": 0.202232, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "zai/glm-4.5": { "accuracy": 33.333, "latency": 245.339, "stderr": 9.622, "cost_per_test": 0.326034, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "google/gemini-2.5-pro": { "accuracy": 33.333, "latency": 620.788, "stderr": 9.622, "cost_per_test": 1.305929, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 33.333, "latency": 694.702, "stderr": 9.622, "cost_per_test": 0.489165, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3p1": { "accuracy": 33.333, "latency": 845.269, "stderr": 9.622, "cost_per_test": 0.595718, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 33.333, "latency": 1011.093, "stderr": 9.622, "cost_per_test": 0.158423, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 33.333, "latency": 1160.238, "stderr": 9.622, "cost_per_test": 3.123072, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-4.6": { "accuracy": 29.167, "latency": 192.546, "stderr": 9.278, "cost_per_test": 0.479308, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "mistralai/devstral-2512": { "accuracy": 29.167, "latency": 379.749, "stderr": 9.278, "cost_per_test": 0.667237, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "google/gemini-3-pro-preview": { "accuracy": 29.167, "latency": 768.965, "stderr": 9.278, "cost_per_test": 0.517236, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.1-codex-max": { "accuracy": 29.167, "latency": 1500.932, "stderr": 9.278, "cost_per_test": 2.2501, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/labs-devstral-small-2512": { "accuracy": 25.0, "latency": 480.823, "stderr": 8.839, "cost_per_test": 0.252954, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-5-2025-08-07": { "accuracy": 25.0, "latency": 1241.634, "stderr": 8.839, "cost_per_test": 0.110631, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/kimi-k2-instruct-0905": { "accuracy": 20.833, "latency": 198.754, "stderr": 8.29, "cost_per_test": 0.597529, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "kimi/kimi-k2-thinking": { "accuracy": 20.833, "latency": 202.742, "stderr": 8.29, "cost_per_test": 0.578583, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 20.833, "latency": 202.742, "stderr": 8.29, "cost_per_test": 0.578583, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 20.833, "latency": 388.684, "stderr": 8.29, "cost_per_test": 0.018737, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 20.833, "latency": 443.073, "stderr": 8.29, "cost_per_test": 0.065418, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "minimax/MiniMax-M2.1": { "accuracy": 20.833, "latency": 800.279, "stderr": 8.29, "cost_per_test": 0.109204, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "grok/grok-4-0709": { "accuracy": 20.833, "latency": 1571.29, "stderr": 8.29, "cost_per_test": 11.376304, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "alibaba/qwen3-max-preview": { "accuracy": 16.667, "latency": 281.614, "stderr": 7.607, "cost_per_test": 1.331133, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "google/gemini-2.5-flash-thinking": { "accuracy": 16.667, "latency": 408.434, "stderr": 7.607, "cost_per_test": 0.798226, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3-max": { "accuracy": 16.667, "latency": 409.732, "stderr": 7.607, "cost_per_test": 0.640975, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 16.667, "latency": 2426.24, "stderr": 7.607, "cost_per_test": 0.693855, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 12.5, "latency": 102.85, "stderr": 6.751, "cost_per_test": 0.18865, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 12.5, "latency": 139.502, "stderr": 6.751, "cost_per_test": 0.195869, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 12.5, "latency": 225.401, "stderr": 6.751, "cost_per_test": 0.037094, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-fast-reasoning": { "accuracy": 12.5, "latency": 255.66, "stderr": 6.751, "cost_per_test": 0.31753, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 12.5, "latency": 353.193, "stderr": 6.751, "cost_per_test": 1.889297, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/magistral-medium-2509": { "accuracy": 12.5, "latency": 901.113, "stderr": 6.751, "cost_per_test": 1.301146, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/gpt-oss-120b": { "accuracy": 8.333, "latency": 47.352, "stderr": 5.642, "cost_per_test": 0.021621, "temperature": null, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-code-fast-1": { "accuracy": 8.333, "latency": 150.138, "stderr": 5.642, "cost_per_test": 0.485227, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-large-2512": { "accuracy": 8.333, "latency": 209.731, "stderr": 5.642, "cost_per_test": 0.078137, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/magistral-small-2509": { "accuracy": 8.333, "latency": 539.759, "stderr": 5.642, "cost_per_test": 0.143668, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 4.167, "latency": 72.477, "stderr": 4.079, "cost_per_test": 0.053975, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "cohere/command-a-03-2025": { "accuracy": 4.167, "latency": 507.73, "stderr": 4.079, "cost_per_test": 3.701636, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 0.0, "latency": 69.204, "stderr": 0.0, "cost_per_test": 0.361917, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-r1": { "accuracy": 0.0, "latency": 171.873, "stderr": 0.0, "cost_per_test": 0.329006, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "ai21labs/jamba-large-1.7": { "accuracy": 0.0, "latency": 879.725, "stderr": 0.0, "cost_per_test": 6.364201, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } } } }