{ "metadata": { "benchmark": "Terminal-Bench 2.0", "slug": "terminal-bench-2", "description": "State-of-the-art set of difficult terminal-based tasks", "benchmark_id": "terminal_bench_2", "family": "terminal_bench", "version": "2.0", "updated": "2026-06-04", "dataset_type": "public", "industry": "coding", "tasks": { "overall": "Overall", "easy": "Easy", "medium": "Medium", "hard": "Hard" }, "models": [ "alibaba/qwen3-max", "alibaba/qwen3-max-2026-01-23", "alibaba/qwen3.5-flash", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-27b", "alibaba/qwen3.6-max-preview", "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "arcee-ai/trinity-large-thinking", "cohere/command-a-03-2025", "deepseek/deepseek-v4-pro", "fireworks/deepseek-v3p1", "fireworks/deepseek-v3p2", "fireworks/deepseek-v3p2-thinking", "fireworks/gpt-oss-120b", "fireworks/llama4-maverick-instruct-basic", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-pro", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemma-4-31b-it", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "kimi/kimi-k2-thinking", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "meta/muse_spark", "minimax/MiniMax-M2.1", "minimax/MiniMax-M2.5", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/magistral-medium-2509", "mistralai/mistral-large-2512", "mistralai/mistral-medium-3.5", "mistralai/mistral-small-2603", "openai/gpt-4.1-2025-04-14", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.3-codex", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "poolside/laguna-m.1", "poolside/laguna-xs.2", "together/moonshotai/Kimi-K2-Instruct", "zai/glm-4.6", "zai/glm-4.7", "zai/glm-5-thinking", "zai/glm-5.1" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "total_models": 67 }, "tasks": { "overall": { "openai/gpt-5.5": { "accuracy": 73.202, "latency": 2298.224, "stderr": 3.978, "cost_per_test": 1.819086, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 70.037, "latency": 472.445, "stderr": 1.35, "cost_per_test": 0.883864, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 68.539, "latency": 660.89, "stderr": 4.95, "cost_per_test": 0.975907, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 67.416, "latency": 408.048, "stderr": 1.297, "cost_per_test": 1.192081, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 67.416, "latency": 542.887, "stderr": 4.996, "cost_per_test": 0.516084, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 64.045, "latency": 506.283, "stderr": 5.115, "cost_per_test": 0.345045, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 59.551, "latency": 566.325, "stderr": 5.232, "cost_per_test": 0.070242, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 59.551, "latency": 615.5, "stderr": 5.232, "cost_per_test": 0.518966, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 59.176, "latency": 664.843, "stderr": 1.35, "cost_per_test": 0.719164, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 58.427, "latency": 627.493, "stderr": 5.254, "cost_per_test": 1.175583, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 58.427, "latency": 898.494, "stderr": 5.254, "cost_per_test": 1.140327, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 58.427, "latency": 1027.517, "stderr": 5.254, "cost_per_test": 0.511959, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 57.303, "latency": 795.458, "stderr": 5.273, "cost_per_test": 0.192786, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 56.18, "latency": 828.342, "stderr": 5.289, "cost_per_test": 0.179637, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 55.056, "latency": 416.249, "stderr": 5.303, "cost_per_test": 0.455782, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 53.933, "latency": 661.913, "stderr": 5.314, "cost_per_test": 1.122133, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 53.933, "latency": 881.675, "stderr": 5.314, "cost_per_test": 0.158286, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 51.685, "latency": 456.071, "stderr": 5.327, "cost_per_test": 0.168702, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 51.685, "latency": 742.684, "stderr": 5.327, "cost_per_test": 0.469958, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-max-preview": { "accuracy": 51.685, "latency": 897.845, "stderr": 5.327, "cost_per_test": 0.320393, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5-thinking": { "accuracy": 49.438, "latency": 875.602, "stderr": 5.33, "cost_per_test": 0.247234, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 47.191, "latency": 839.048, "stderr": 5.322, "cost_per_test": 0.073995, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 46.067, "latency": 1019.457, "stderr": 5.314, "cost_per_test": 0.216209, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 44.944, "latency": 840.666, "stderr": 5.303, "cost_per_test": 0.14597, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 44.944, "latency": 886.169, "stderr": 5.303, "cost_per_test": 0.407628, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 44.944, "latency": 2007.631, "stderr": 5.303, "cost_per_test": 0.298145, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 44.944, "latency": 2241.556, "stderr": 4.492, "cost_per_test": 0.914107, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 43.446, "latency": 1954.824, "stderr": 4.406, "cost_per_test": 1.115571, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 41.573, "latency": 736.918, "stderr": 5.254, "cost_per_test": 0.751119, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 41.573, "latency": 754.513, "stderr": 5.254, "cost_per_test": 0.085163, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 41.573, "latency": 1877.151, "stderr": 5.254, "cost_per_test": 0.321994, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 40.449, "latency": 161.408, "stderr": 5.232, "cost_per_test": 0.154109, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 40.449, "latency": 756.405, "stderr": 5.232, "cost_per_test": 0.139997, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 39.888, "latency": 2527.932, "stderr": 4.454, "cost_per_test": 0.224388, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 39.326, "latency": 855.298, "stderr": 5.207, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 38.202, "latency": 655.044, "stderr": 5.18, "cost_per_test": 0.327722, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.7": { "accuracy": 38.202, "latency": 777.512, "stderr": 5.18, "cost_per_test": 0.183983, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 37.079, "latency": 782.07, "stderr": 5.149, "cost_per_test": 0.070371, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 37.079, "latency": 986.102, "stderr": 5.149, "cost_per_test": 0.477064, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 37.079, "latency": 986.553, "stderr": 5.149, "cost_per_test": 0.115876, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 35.955, "latency": 1024.966, "stderr": 5.115, "cost_per_test": 0.166362, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 34.831, "latency": 804.541, "stderr": 5.079, "cost_per_test": 0.063298, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 31.461, "latency": 0.0, "stderr": 4.95, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 30.337, "latency": 796.526, "stderr": 4.901, "cost_per_test": 0.472003, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 30.337, "latency": 1253.133, "stderr": 4.901, "cost_per_test": 4.710448, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 29.213, "latency": 431.268, "stderr": 4.848, "cost_per_test": 0.03431, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 28.09, "latency": 0.0, "stderr": 4.791, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "zai/glm-4.6": { "accuracy": 28.09, "latency": 576.085, "stderr": 4.791, "cost_per_test": 0.14355, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-0709": { "accuracy": 28.09, "latency": 885.185, "stderr": 4.791, "cost_per_test": 0.552755, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 26.966, "latency": 1028.621, "stderr": 4.731, "cost_per_test": 0.116731, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 25.843, "latency": 702.2, "stderr": 4.667, "cost_per_test": 1.107135, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 24.719, "latency": 479.157, "stderr": 4.599, "cost_per_test": 0.055526, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 24.719, "latency": 667.879, "stderr": 4.599, "cost_per_test": 0.065496, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 24.719, "latency": 716.499, "stderr": 4.599, "cost_per_test": 0.110414, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max": { "accuracy": 24.719, "latency": 912.424, "stderr": 4.599, "cost_per_test": 0.423036, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p1": { "accuracy": 22.472, "latency": 837.847, "stderr": 4.449, "cost_per_test": 0.501536, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 21.348, "latency": 417.621, "stderr": 4.368, "cost_per_test": 0.143966, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 20.225, "latency": 831.981, "stderr": 4.282, "cost_per_test": 0.263469, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 19.101, "latency": 490.948, "stderr": 4.19, "cost_per_test": 0.131646, "temperature": null, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 17.978, "latency": 253.172, "stderr": 4.093, "cost_per_test": 0.032811, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "arcee-ai/trinity-large-thinking": { "accuracy": 17.978, "latency": 1015.224, "stderr": 4.093, "cost_per_test": 0.325546, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Arcee-Ai", "harness": null }, "mistralai/mistral-small-2603": { "accuracy": 16.854, "latency": 407.865, "stderr": 3.991, "cost_per_test": 0.105762, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 14.607, "latency": 589.207, "stderr": 3.765, "cost_per_test": 1.078347, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 13.483, "latency": 898.203, "stderr": 3.641, "cost_per_test": 0.567192, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 8.989, "latency": 1180.302, "stderr": 3.049, "cost_per_test": 0.030482, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 2.247, "latency": 760.674, "stderr": 1.58, "cost_per_test": 2.436871, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 2.247, "latency": 1193.58, "stderr": 1.58, "cost_per_test": 0.03402, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "easy": { "grok/grok-4.20-0309-reasoning": { "accuracy": 100.0, "latency": 3591.339, "stderr": 0.0, "cost_per_test": 3.428921, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 100.0, "latency": 9079.061, "stderr": 0.0, "cost_per_test": 40.526503, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 100.0, "latency": 9595.719, "stderr": 0.0, "cost_per_test": 0.7634, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 100.0, "latency": 10147.584, "stderr": 0.0, "cost_per_test": 3.753614, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 100.0, "latency": 11264.79, "stderr": 0.0, "cost_per_test": 7.677245, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 100.0, "latency": 12079.231, "stderr": 0.0, "cost_per_test": 11.482876, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 100.0, "latency": 13961.721, "stderr": 0.0, "cost_per_test": 26.15673, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 100.0, "latency": 14574.739, "stderr": 0.0, "cost_per_test": 7.291809, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 100.0, "latency": 14704.807, "stderr": 0.0, "cost_per_test": 21.713924, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 100.0, "latency": 14727.563, "stderr": 0.0, "cost_per_test": 24.967451, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 100.0, "latency": 17722.698, "stderr": 0.0, "cost_per_test": 10.502075, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 100.0, "latency": 19991.481, "stderr": 0.0, "cost_per_test": 25.372269, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 100.0, "latency": 22862.255, "stderr": 0.0, "cost_per_test": 9.975575, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 91.667, "latency": 10511.909, "stderr": 8.333, "cost_per_test": 11.799576, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 85.0, "latency": 51135.488, "stderr": 15.0, "cost_per_test": 84.073356, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 83.333, "latency": 14792.759, "stderr": 8.333, "cost_per_test": 16.001404, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "poolside/laguna-m.1": { "accuracy": 75.0, "latency": 0.0, "stderr": 25.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 75.0, "latency": 0.0, "stderr": 25.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 75.0, "latency": 9261.539, "stderr": 25.0, "cost_per_test": 10.141149, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 75.0, "latency": 9292.061, "stderr": 25.0, "cost_per_test": 3.203245, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 75.0, "latency": 10661.252, "stderr": 25.0, "cost_per_test": 1.235461, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark": { "accuracy": 75.0, "latency": 12600.741, "stderr": 25.0, "cost_per_test": 1.562883, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-4.6": { "accuracy": 75.0, "latency": 12817.898, "stderr": 25.0, "cost_per_test": 3.193998, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 75.0, "latency": 13694.866, "stderr": 25.0, "cost_per_test": 11.546985, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 75.0, "latency": 16396.435, "stderr": 25.0, "cost_per_test": 16.712406, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 75.0, "latency": 16524.711, "stderr": 25.0, "cost_per_test": 10.456572, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 75.0, "latency": 16787.905, "stderr": 25.0, "cost_per_test": 1.894882, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 75.0, "latency": 16830.017, "stderr": 25.0, "cost_per_test": 3.114944, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-4.7": { "accuracy": 75.0, "latency": 17299.642, "stderr": 25.0, "cost_per_test": 4.093619, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 75.0, "latency": 17698.932, "stderr": 25.0, "cost_per_test": 4.289484, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 75.0, "latency": 17901.029, "stderr": 25.0, "cost_per_test": 1.408372, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 75.0, "latency": 18430.614, "stderr": 25.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 75.0, "latency": 18668.818, "stderr": 25.0, "cost_per_test": 1.646383, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 75.0, "latency": 18704.807, "stderr": 25.0, "cost_per_test": 3.247824, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 75.0, "latency": 19030.377, "stderr": 25.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5-thinking": { "accuracy": 75.0, "latency": 19482.139, "stderr": 25.0, "cost_per_test": 5.500962, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.1": { "accuracy": 75.0, "latency": 19617.26, "stderr": 25.0, "cost_per_test": 3.521858, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-0709": { "accuracy": 75.0, "latency": 19695.363, "stderr": 25.0, "cost_per_test": 12.298802, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 75.0, "latency": 19717.265, "stderr": 25.0, "cost_per_test": 9.069724, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-max-preview": { "accuracy": 75.0, "latency": 19977.046, "stderr": 25.0, "cost_per_test": 7.12874, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max": { "accuracy": 75.0, "latency": 20301.426, "stderr": 25.0, "cost_per_test": 9.412554, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 75.0, "latency": 21940.775, "stderr": 25.0, "cost_per_test": 10.61468, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 75.0, "latency": 22886.816, "stderr": 25.0, "cost_per_test": 2.597261, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 75.0, "latency": 27882.215, "stderr": 25.0, "cost_per_test": 104.807458, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 75.0, "latency": 41766.603, "stderr": 25.0, "cost_per_test": 7.164356, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 75.0, "latency": 44669.792, "stderr": 25.0, "cost_per_test": 6.633727, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 75.0, "latency": 49874.621, "stderr": 25.0, "cost_per_test": 20.338889, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 75.0, "latency": 56246.478, "stderr": 25.0, "cost_per_test": 4.992625, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 66.667, "latency": 43494.837, "stderr": 23.57, "cost_per_test": 24.821464, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 50.0, "latency": 5633.071, "stderr": 28.868, "cost_per_test": 0.730044, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-small-2603": { "accuracy": 50.0, "latency": 9074.999, "stderr": 28.868, "cost_per_test": 2.353195, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 50.0, "latency": 10923.582, "stderr": 28.868, "cost_per_test": 2.929132, "temperature": null, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 50.0, "latency": 13109.85, "stderr": 28.868, "cost_per_test": 23.993219, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 50.0, "latency": 14860.307, "stderr": 28.868, "cost_per_test": 0.46031, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 50.0, "latency": 15623.94, "stderr": 28.868, "cost_per_test": 24.633748, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 50.0, "latency": 15942.11, "stderr": 28.868, "cost_per_test": 2.456715, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 50.0, "latency": 17401.069, "stderr": 28.868, "cost_per_test": 1.56576, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 50.0, "latency": 18511.579, "stderr": 28.868, "cost_per_test": 5.862182, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p1": { "accuracy": 50.0, "latency": 18642.103, "stderr": 28.868, "cost_per_test": 11.159169, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 50.0, "latency": 19985.016, "stderr": 28.868, "cost_per_test": 12.620022, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 50.0, "latency": 21950.812, "stderr": 28.868, "cost_per_test": 2.578242, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "arcee-ai/trinity-large-thinking": { "accuracy": 50.0, "latency": 22588.728, "stderr": 28.868, "cost_per_test": 7.243399, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Arcee-Ai", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 50.0, "latency": 22682.927, "stderr": 28.868, "cost_per_test": 4.810641, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 50.0, "latency": 22805.496, "stderr": 28.868, "cost_per_test": 3.70156, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 50.0, "latency": 26261.72, "stderr": 28.868, "cost_per_test": 0.678233, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 25.0, "latency": 26557.161, "stderr": 25.0, "cost_per_test": 0.75695, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 0.0, "latency": 16924.986, "stderr": 0.0, "cost_per_test": 54.220382, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null } }, "medium": { "openai/gpt-5.5": { "accuracy": 79.273, "latency": 3718.945, "stderr": 4.641, "cost_per_test": 6.114426, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 76.97, "latency": 764.502, "stderr": 1.212, "cost_per_test": 0.858151, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 72.727, "latency": 660.295, "stderr": 2.777, "cost_per_test": 2.947382, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 72.727, "latency": 878.489, "stderr": 6.061, "cost_per_test": 0.835118, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 72.727, "latency": 1069.441, "stderr": 6.061, "cost_per_test": 1.579195, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 70.909, "latency": 1453.926, "stderr": 6.181, "cost_per_test": 1.845256, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 69.091, "latency": 819.257, "stderr": 6.289, "cost_per_test": 0.558345, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 69.091, "latency": 916.418, "stderr": 6.289, "cost_per_test": 0.113664, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 69.091, "latency": 995.99, "stderr": 6.289, "cost_per_test": 0.839781, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 69.091, "latency": 1015.398, "stderr": 6.289, "cost_per_test": 1.902308, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 69.091, "latency": 1662.709, "stderr": 6.289, "cost_per_test": 0.725496, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 67.879, "latency": 1075.837, "stderr": 2.424, "cost_per_test": 1.163738, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 67.273, "latency": 1287.195, "stderr": 6.385, "cost_per_test": 0.311962, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 67.273, "latency": 1340.408, "stderr": 6.385, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.6-max-preview": { "accuracy": 67.273, "latency": 1452.876, "stderr": 6.385, "cost_per_test": 0.518454, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 63.636, "latency": 673.567, "stderr": 6.546, "cost_per_test": 0.737538, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 61.818, "latency": 1071.095, "stderr": 6.611, "cost_per_test": 1.815815, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "zai/glm-5-thinking": { "accuracy": 61.818, "latency": 1416.883, "stderr": 6.611, "cost_per_test": 0.40007, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.1": { "accuracy": 61.818, "latency": 1426.71, "stderr": 6.611, "cost_per_test": 0.256135, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 60.0, "latency": 1649.667, "stderr": 6.667, "cost_per_test": 0.349865, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 56.364, "latency": 1357.732, "stderr": 6.749, "cost_per_test": 0.119737, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 56.364, "latency": 3248.712, "stderr": 6.749, "cost_per_test": 0.482453, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 53.939, "latency": 3627.245, "stderr": 5.572, "cost_per_test": 1.479192, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 52.727, "latency": 738.006, "stderr": 6.794, "cost_per_test": 0.27299, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 52.727, "latency": 1220.939, "stderr": 6.794, "cost_per_test": 0.13781, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 52.727, "latency": 1360.35, "stderr": 6.794, "cost_per_test": 0.236205, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 52.727, "latency": 1433.983, "stderr": 6.794, "cost_per_test": 0.659616, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 52.727, "latency": 3037.571, "stderr": 6.794, "cost_per_test": 0.521044, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.3": { "accuracy": 52.121, "latency": 3163.261, "stderr": 5.722, "cost_per_test": 1.805197, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 50.909, "latency": 1201.797, "stderr": 6.803, "cost_per_test": 0.760478, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 49.091, "latency": 261.188, "stderr": 6.803, "cost_per_test": 0.249376, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 49.091, "latency": 1192.468, "stderr": 6.803, "cost_per_test": 1.215448, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 49.091, "latency": 1224.001, "stderr": 6.803, "cost_per_test": 0.226541, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 49.091, "latency": 1265.532, "stderr": 6.803, "cost_per_test": 0.113873, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 47.273, "latency": 1059.981, "stderr": 6.794, "cost_per_test": 0.530313, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 47.273, "latency": 1596.423, "stderr": 6.794, "cost_per_test": 0.187508, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 47.273, "latency": 1658.582, "stderr": 6.794, "cost_per_test": 0.269204, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 47.273, "latency": 4090.653, "stderr": 5.652, "cost_per_test": 0.3631, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 45.455, "latency": 1384.027, "stderr": 6.776, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 45.455, "latency": 1595.693, "stderr": 6.776, "cost_per_test": 0.771977, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.7": { "accuracy": 43.636, "latency": 1258.156, "stderr": 6.749, "cost_per_test": 0.297718, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 43.636, "latency": 1301.893, "stderr": 6.749, "cost_per_test": 0.102427, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 41.818, "latency": 2027.797, "stderr": 6.712, "cost_per_test": 7.622361, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 38.182, "latency": 0.0, "stderr": 6.611, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 38.182, "latency": 1288.924, "stderr": 6.611, "cost_per_test": 0.763787, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-0709": { "accuracy": 36.364, "latency": 1432.39, "stderr": 6.546, "cost_per_test": 0.894458, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 34.545, "latency": 0.0, "stderr": 6.471, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "zai/glm-4.6": { "accuracy": 34.545, "latency": 932.211, "stderr": 6.471, "cost_per_test": 0.232291, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 34.545, "latency": 1080.75, "stderr": 6.471, "cost_per_test": 0.033477, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 32.727, "latency": 697.87, "stderr": 6.385, "cost_per_test": 0.05552, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 32.727, "latency": 775.364, "stderr": 6.385, "cost_per_test": 0.089852, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 30.909, "latency": 1159.426, "stderr": 6.289, "cost_per_test": 0.17867, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p1": { "accuracy": 29.091, "latency": 1355.789, "stderr": 6.181, "cost_per_test": 0.811576, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 29.091, "latency": 1476.467, "stderr": 6.181, "cost_per_test": 0.684549, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 29.091, "latency": 1664.496, "stderr": 6.181, "cost_per_test": 0.188892, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 27.273, "latency": 1136.287, "stderr": 6.061, "cost_per_test": 1.791545, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 25.455, "latency": 675.786, "stderr": 5.928, "cost_per_test": 0.232963, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 21.818, "latency": 409.678, "stderr": 5.62, "cost_per_test": 0.053094, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-small-2603": { "accuracy": 21.818, "latency": 660.0, "stderr": 5.62, "cost_per_test": 0.171141, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 21.818, "latency": 1346.297, "stderr": 5.62, "cost_per_test": 0.426341, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 20.0, "latency": 794.442, "stderr": 5.443, "cost_per_test": 0.213028, "temperature": null, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "arcee-ai/trinity-large-thinking": { "accuracy": 20.0, "latency": 1642.817, "stderr": 5.443, "cost_per_test": 0.526793, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Arcee-Ai", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 18.182, "latency": 953.444, "stderr": 5.249, "cost_per_test": 1.744961, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 18.182, "latency": 1453.456, "stderr": 5.249, "cost_per_test": 0.91782, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 10.909, "latency": 1909.943, "stderr": 4.242, "cost_per_test": 0.049326, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 1.818, "latency": 1230.908, "stderr": 1.818, "cost_per_test": 3.943301, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 1.818, "latency": 1931.43, "stderr": 1.818, "cost_per_test": 0.055051, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "hard": { "openai/gpt-5.5": { "accuracy": 60.5, "latency": 6818.065, "stderr": 7.567, "cost_per_test": 11.209781, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 56.667, "latency": 1960.641, "stderr": 9.202, "cost_per_test": 2.89519, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 54.444, "latency": 1401.588, "stderr": 2.94, "cost_per_test": 1.573277, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 53.333, "latency": 1210.541, "stderr": 3.333, "cost_per_test": 5.403534, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 53.333, "latency": 1610.564, "stderr": 9.264, "cost_per_test": 1.53105, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 50.0, "latency": 1501.972, "stderr": 9.285, "cost_per_test": 1.023633, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 50.0, "latency": 2203.295, "stderr": 9.285, "cost_per_test": 1.39421, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 43.333, "latency": 1353.011, "stderr": 9.202, "cost_per_test": 0.500482, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark": { "accuracy": 40.0, "latency": 1680.099, "stderr": 9.097, "cost_per_test": 0.208384, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 40.0, "latency": 1825.982, "stderr": 9.097, "cost_per_test": 1.539598, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 40.0, "latency": 1972.368, "stderr": 0.0, "cost_per_test": 2.133521, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 36.667, "latency": 1234.872, "stderr": 8.949, "cost_per_test": 1.352153, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 36.667, "latency": 2359.858, "stderr": 8.949, "cost_per_test": 0.571931, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.1": { "accuracy": 36.667, "latency": 2615.635, "stderr": 8.949, "cost_per_test": 0.469581, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 33.333, "latency": 1861.563, "stderr": 8.754, "cost_per_test": 3.487564, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 33.333, "latency": 1963.675, "stderr": 8.754, "cost_per_test": 3.328994, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 33.333, "latency": 2457.415, "stderr": 8.754, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 33.333, "latency": 3048.301, "stderr": 8.754, "cost_per_test": 1.330077, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 30.0, "latency": 2665.531, "stderr": 8.51, "cost_per_test": 3.382969, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 26.667, "latency": 2489.176, "stderr": 8.212, "cost_per_test": 0.219518, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 26.667, "latency": 2493.974, "stderr": 8.212, "cost_per_test": 0.433043, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 26.667, "latency": 2628.969, "stderr": 8.212, "cost_per_test": 1.209297, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 24.444, "latency": 5799.312, "stderr": 5.966, "cost_per_test": 3.309529, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 24.444, "latency": 6649.949, "stderr": 6.577, "cost_per_test": 2.711852, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 23.333, "latency": 2186.191, "stderr": 7.854, "cost_per_test": 2.228321, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.7": { "accuracy": 23.333, "latency": 2306.619, "stderr": 7.854, "cost_per_test": 0.545816, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 23.333, "latency": 2537.384, "stderr": 7.854, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5-thinking": { "accuracy": 23.333, "latency": 2597.618, "stderr": 7.854, "cost_per_test": 0.733462, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 21.667, "latency": 7499.53, "stderr": 6.35, "cost_per_test": 0.665683, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 20.0, "latency": 2083.192, "stderr": 7.428, "cost_per_test": 3.2845, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 20.0, "latency": 2244.002, "stderr": 7.428, "cost_per_test": 0.415326, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.6-max-preview": { "accuracy": 20.0, "latency": 2663.606, "stderr": 7.428, "cost_per_test": 0.950499, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 20.0, "latency": 3024.39, "stderr": 7.428, "cost_per_test": 0.641419, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 20.0, "latency": 5955.972, "stderr": 7.428, "cost_per_test": 0.884497, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 16.667, "latency": 478.845, "stderr": 6.92, "cost_per_test": 0.457189, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 16.667, "latency": 2238.387, "stderr": 6.92, "cost_per_test": 0.252651, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 16.667, "latency": 2925.437, "stderr": 6.92, "cost_per_test": 1.415291, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 16.667, "latency": 2926.775, "stderr": 6.92, "cost_per_test": 0.343766, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 16.667, "latency": 3051.575, "stderr": 6.92, "cost_per_test": 0.346302, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 16.667, "latency": 5568.88, "stderr": 6.92, "cost_per_test": 0.955247, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "poolside/laguna-m.1": { "accuracy": 13.333, "latency": 0.0, "stderr": 6.312, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 13.333, "latency": 1279.429, "stderr": 6.312, "cost_per_test": 0.101787, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 13.333, "latency": 1456.478, "stderr": 6.312, "cost_per_test": 0.390551, "temperature": null, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 13.333, "latency": 1943.298, "stderr": 6.312, "cost_per_test": 0.972241, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 13.333, "latency": 2320.142, "stderr": 6.312, "cost_per_test": 0.208768, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 13.333, "latency": 2386.804, "stderr": 6.312, "cost_per_test": 0.187783, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 13.333, "latency": 2468.211, "stderr": 6.312, "cost_per_test": 0.781624, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 13.333, "latency": 3040.733, "stderr": 6.312, "cost_per_test": 0.493541, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 10.0, "latency": 0.0, "stderr": 5.571, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "zai/glm-4.6": { "accuracy": 10.0, "latency": 1709.053, "stderr": 5.571, "cost_per_test": 0.425866, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 10.0, "latency": 2125.615, "stderr": 5.571, "cost_per_test": 0.327562, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max": { "accuracy": 10.0, "latency": 2706.857, "stderr": 5.571, "cost_per_test": 1.255007, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "arcee-ai/trinity-large-thinking": { "accuracy": 10.0, "latency": 3011.83, "stderr": 5.571, "cost_per_test": 0.965787, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Arcee-Ai", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 6.667, "latency": 751.076, "stderr": 4.632, "cost_per_test": 0.097339, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 6.667, "latency": 1238.941, "stderr": 4.632, "cost_per_test": 0.427099, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 6.667, "latency": 2363.026, "stderr": 4.632, "cost_per_test": 1.400277, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-v3p1": { "accuracy": 6.667, "latency": 2485.614, "stderr": 4.632, "cost_per_test": 1.487889, "temperature": 1.0, "top_p": null, "max_output_tokens": 163840, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-0709": { "accuracy": 6.667, "latency": 2626.048, "stderr": 4.632, "cost_per_test": 1.63984, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-small-2603": { "accuracy": 3.333, "latency": 1210.0, "stderr": 3.333, "cost_per_test": 0.313759, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 3.333, "latency": 1421.5, "stderr": 3.333, "cost_per_test": 0.164728, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 3.333, "latency": 1747.98, "stderr": 3.333, "cost_per_test": 3.199096, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 3.333, "latency": 1981.374, "stderr": 3.333, "cost_per_test": 0.061375, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 3.333, "latency": 2256.665, "stderr": 3.333, "cost_per_test": 7.229384, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 3.333, "latency": 3717.629, "stderr": 3.333, "cost_per_test": 13.974328, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 0.0, "latency": 2664.669, "stderr": 0.0, "cost_per_test": 1.68267, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 0.0, "latency": 3501.563, "stderr": 0.0, "cost_per_test": 0.090431, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 0.0, "latency": 3540.955, "stderr": 0.0, "cost_per_test": 0.100927, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } } } }