{ "metadata": { "benchmark": "AIME", "slug": "aime", "description": "Challenging national math exam given to top high-school students", "benchmark_id": "aime", "family": "aime", "version": "1", "updated": "2026-04-16", "dataset_type": "public", "industry": "math", "tasks": { "overall": "Overall", "aime_2024": "AIME 2024", "aime_2025": "AIME 2025" }, "models": [ "ai21labs/jamba-large-1.6", "ai21labs/jamba-mini-1.6", "alibaba/qwen3-max", "alibaba/qwen3-max-preview", "alibaba/qwen3.5-flash", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-plus", "anthropic/claude-3-5-haiku-20241022", "anthropic/claude-3-5-sonnet-20241022", "anthropic/claude-3-7-sonnet-20250219", "anthropic/claude-3-7-sonnet-20250219-thinking", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-1-20250805-thinking", "anthropic/claude-opus-4-20250514", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "cohere/command-a-03-2025", "fireworks/deepseek-r1", "fireworks/deepseek-v3", "fireworks/deepseek-v3-0324", "fireworks/deepseek-v3p2", "fireworks/deepseek-v3p2-thinking", "fireworks/gpt-oss-120b", "fireworks/gpt-oss-20b", "fireworks/llama4-maverick-instruct-basic", "fireworks/qwen3-235b-a22b", "google/gemini-1.5-flash-002", "google/gemini-1.5-pro-002", "google/gemini-2.0-flash-001", "google/gemini-2.5-flash-lite-preview-09-2025", "google/gemini-2.5-flash-lite-preview-09-2025-thinking", "google/gemini-2.5-flash-preview-09-2025", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-pro-exp-03-25", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "grok/grok-2-1212", "grok/grok-3", "grok/grok-3-mini-fast-high-reasoning", "grok/grok-3-mini-fast-low-reasoning", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "kimi/kimi-k2-thinking", "kimi/kimi-k2.5-thinking", "meta/muse_spark", "minimax/MiniMax-M2.1", "minimax/MiniMax-M2.5", "minimax/MiniMax-M2.7", "mistralai/magistral-medium-2509", "mistralai/magistral-small-2509", "mistralai/mistral-large-2411", "mistralai/mistral-large-2512", "mistralai/mistral-medium-2505", "mistralai/mistral-small-2402", "mistralai/mistral-small-2503", "openai/gpt-4.1-2025-04-14", "openai/gpt-4.1-mini-2025-04-14", "openai/gpt-4.1-nano-2025-04-14", "openai/gpt-4o-2024-08-06", "openai/gpt-4o-2024-11-20", "openai/gpt-4o-mini-2024-07-18", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-nano-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/o1-2024-12-17", "openai/o3-2025-04-16", "openai/o3-mini-2025-01-31", "openai/o4-mini-2025-04-16", "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561", "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking", "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "together/meta-llama/Llama-4-Scout-17B-16E-Instruct", "together/moonshotai/Kimi-K2-Instruct", "zai/glm-4.5", "zai/glm-4.6", "zai/glm-4.7", "zai/glm-5-thinking", "zai/glm-5.1" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "platform", "mode": "one-shot", "archived": true, "total_models": 96 }, "tasks": { "overall": { "google/gemini-3.1-pro-preview": { "accuracy": 98.125, "latency": 104.525, "stderr": 0.343, "cost_per_test": 0.126991, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 96.875, "latency": 120.31, "stderr": 0.208, "cost_per_test": 0.135323, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "meta/muse_spark": { "accuracy": 96.875, "latency": 156.746, "stderr": 0.378, "cost_per_test": 0.004561, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta" }, "google/gemini-3-pro-preview": { "accuracy": 96.68, "latency": 109.264, "stderr": 0.384, "cost_per_test": 0.169306, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.4-2026-03-05": { "accuracy": 96.667, "latency": 121.466, "stderr": 0.534, "cost_per_test": 0.138186, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 96.458, "latency": 106.799, "stderr": 0.516, "cost_per_test": 0.059397, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-opus-4-7": { "accuracy": 96.25, "latency": 182.065, "stderr": 0.522, "cost_per_test": 0.450513, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3-flash-preview": { "accuracy": 95.625, "latency": 84.007, "stderr": 0.516, "cost_per_test": 0.046328, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 95.625, "latency": 295.678, "stderr": 0.644, "cost_per_test": 0.343821, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 95.625, "latency": 300.051, "stderr": 0.409, "cost_per_test": 0.128636, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2.5-thinking": { "accuracy": 95.625, "latency": 322.611, "stderr": 0.343, "cost_per_test": 0.064142, "temperature": null, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 95.417, "latency": 212.201, "stderr": 0.369, "cost_per_test": 1.320246, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "alibaba/qwen3.6-plus": { "accuracy": 94.583, "latency": 516.49, "stderr": 0.579, "cost_per_test": 0.09352, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5-2025-08-07": { "accuracy": 93.374, "latency": 292.202, "stderr": 1.305, "cost_per_test": 0.092693, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 93.333, "latency": 163.891, "stderr": 0.62, "cost_per_test": 0.150354, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "zai/glm-4.7": { "accuracy": 93.333, "latency": 529.192, "stderr": 0.445, "cost_per_test": 0.043308, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "zai/glm-4.6": { "accuracy": 92.708, "latency": 699.868, "stderr": 0.605, "cost_per_test": 0.116286, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "fireworks/gpt-oss-120b": { "accuracy": 92.598, "latency": 156.629, "stderr": 1.225, "cost_per_test": 0.010472, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "alibaba/qwen3.5-flash": { "accuracy": 92.5, "latency": 239.603, "stderr": 0.891, "cost_per_test": 0.011117, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "anthropic/claude-sonnet-4-6": { "accuracy": 92.292, "latency": 328.034, "stderr": 0.438, "cost_per_test": 0.436012, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 91.875, "latency": 91.997, "stderr": 0.584, "cost_per_test": 0.004094, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "zai/glm-5.1": { "accuracy": 91.875, "latency": 375.201, "stderr": 0.438, "cost_per_test": 0.070948, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "zai/glm-5-thinking": { "accuracy": 91.667, "latency": 458.694, "stderr": 0.772, "cost_per_test": 0.069817, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 91.458, "latency": 105.53, "stderr": 0.836, "cost_per_test": 0.02276, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-fast-reasoning": { "accuracy": 91.25, "latency": 34.559, "stderr": 1.033, "cost_per_test": 0.004317, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "minimax/MiniMax-M2.7": { "accuracy": 91.042, "latency": 236.637, "stderr": 0.734, "cost_per_test": 0.022959, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "grok/grok-4-0709": { "accuracy": 90.556, "latency": 133.228, "stderr": 1.757, "cost_per_test": 0.174399, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 88.75, "latency": 65.876, "stderr": 0.99, "cost_per_test": 0.003432, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "minimax/MiniMax-M2.5": { "accuracy": 88.75, "latency": 373.487, "stderr": 0.877, "cost_per_test": 0.041379, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 88.19, "latency": 217.504, "stderr": 0.686, "cost_per_test": 0.266927, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-4.5": { "accuracy": 86.667, "latency": 333.117, "stderr": 1.332, "cost_per_test": 0.04689, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "openai/o3-mini-2025-01-31": { "accuracy": 86.458, "latency": 154.65, "stderr": 1.054, "cost_per_test": 0.050817, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 86.042, "latency": 213.536, "stderr": 1.111, "cost_per_test": 0.04516, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "fireworks/gpt-oss-20b": { "accuracy": 86.042, "latency": 244.712, "stderr": 1.133, "cost_per_test": 0.011043, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 85.833, "latency": 143.907, "stderr": 1.318, "cost_per_test": 0.027139, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "kimi/kimi-k2-thinking": { "accuracy": 85.417, "latency": 924.913, "stderr": 1.327, "cost_per_test": 0.044508, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "openai/o3-2025-04-16": { "accuracy": 85.278, "latency": 266.179, "stderr": 1.389, "cost_per_test": 0.060099, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 85.0, "latency": 102.246, "stderr": 0.534, "cost_per_test": 0.008281, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 84.583, "latency": 373.072, "stderr": 1.125, "cost_per_test": 0.00442, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "fireworks/qwen3-235b-a22b": { "accuracy": 83.958, "latency": 242.174, "stderr": 0.768, "cost_per_test": 0.011598, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o4-mini-2025-04-16": { "accuracy": 83.667, "latency": 55.109, "stderr": 1.178, "cost_per_test": 0.025707, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/magistral-medium-2509": { "accuracy": 83.542, "latency": 401.007, "stderr": 1.018, "cost_per_test": 0.086908, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 83.333, "latency": 40.564, "stderr": 1.178, "cost_per_test": 0.006966, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 82.708, "latency": 158.25, "stderr": 0.915, "cost_per_test": 0.173775, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 81.181, "latency": 193.682, "stderr": 1.29, "cost_per_test": 0.008862, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3-max": { "accuracy": 81.042, "latency": 16.76, "stderr": 1.1, "cost_per_test": 0.033364, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "mistralai/magistral-small-2509": { "accuracy": 80.679, "latency": 267.982, "stderr": 1.208, "cost_per_test": 0.025112, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 78.179, "latency": 214.504, "stderr": 1.629, "cost_per_test": 0.85994, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "minimax/MiniMax-M2.1": { "accuracy": 77.917, "latency": 237.635, "stderr": 1.289, "cost_per_test": 0.024152, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 76.875, "latency": 17.442, "stderr": 1.389, "cost_per_test": 0.089253, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 76.25, "latency": 271.788, "stderr": 1.435, "cost_per_test": 0.295026, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-r1": { "accuracy": 73.958, "latency": 153.912, "stderr": 1.343, "cost_per_test": 0.053393, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o1-2024-12-17": { "accuracy": 71.458, "latency": 177.028, "stderr": 1.458, "cost_per_test": 0.658977, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 70.625, "latency": 31.397, "stderr": 0.562, "cost_per_test": 0.00293, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-v3p2": { "accuracy": 64.792, "latency": 119.555, "stderr": 2.046, "cost_per_test": 0.001399, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 62.708, "latency": 124.668, "stderr": 1.217, "cost_per_test": 0.01278, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "alibaba/qwen3-max-preview": { "accuracy": 60.694, "latency": 206.753, "stderr": 2.562, "cost_per_test": 0.023054, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "grok/grok-3": { "accuracy": 58.75, "latency": 63.989, "stderr": 1.439, "cost_per_test": 0.090668, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 53.542, "latency": 167.893, "stderr": 1.596, "cost_per_test": 0.0, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "fireworks/deepseek-v3-0324": { "accuracy": 52.202, "latency": 50.575, "stderr": 1.712, "cost_per_test": 0.003192, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 51.458, "latency": 61.601, "stderr": 1.94, "cost_per_test": 0.004986, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 49.792, "latency": 58.378, "stderr": 2.215, "cost_per_test": 0.004524, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 49.375, "latency": 33.139, "stderr": 1.122, "cost_per_test": 0.0079, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 44.583, "latency": 303.709, "stderr": 1.747, "cost_per_test": 0.395347, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 44.236, "latency": 29.675, "stderr": 1.677, "cost_per_test": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/mistral-large-2512": { "accuracy": 42.917, "latency": 104.325, "stderr": 0.884, "cost_per_test": 0.006086, "temperature": 0.065, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/mistral-medium-2505": { "accuracy": 42.292, "latency": 65.954, "stderr": 1.334, "cost_per_test": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 42.083, "latency": 51.146, "stderr": 1.567, "cost_per_test": 0.000566, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-20250514": { "accuracy": 41.25, "latency": 37.029, "stderr": 1.152, "cost_per_test": 0.110384, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 39.583, "latency": 161.075, "stderr": 1.386, "cost_per_test": 0.050106, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 38.542, "latency": 23.398, "stderr": 0.968, "cost_per_test": 0.02164, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 33.333, "latency": 5.879, "stderr": 1.068, "cost_per_test": 0.000805, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.0-flash-001": { "accuracy": 29.792, "latency": 11.214, "stderr": 0.644, "cost_per_test": 0.001044, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/deepseek-v3": { "accuracy": 27.5, "latency": 58.801, "stderr": 0.73, "cost_per_test": 0.003364, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 27.083, "latency": 17.366, "stderr": 1.935, "cost_per_test": 0.000788, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 26.458, "latency": 11.914, "stderr": 0.814, "cost_per_test": 0.001184, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 26.25, "latency": 18.704, "stderr": 1.989, "cost_per_test": 0.003224, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 25.208, "latency": 15.502, "stderr": 0.784, "cost_per_test": 0.001142, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 22.292, "latency": 18.928, "stderr": 1.352, "cost_per_test": 0.023201, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 18.958, "latency": 21.69, "stderr": 0.7, "cost_per_test": 0.000813, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "google/gemini-1.5-pro-002": { "accuracy": 18.75, "latency": 10.642, "stderr": 0.534, "cost_per_test": 0.004203, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-1.5-flash-002": { "accuracy": 17.292, "latency": 5.696, "stderr": 0.901, "cost_per_test": 0.000266, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 16.042, "latency": 11.244, "stderr": 1.077, "cost_per_test": 0.001136, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "grok/grok-2-1212": { "accuracy": 15.208, "latency": 57.884, "stderr": 0.784, "cost_per_test": 0.015018, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 13.958, "latency": 68.368, "stderr": 0.968, "cost_per_test": 0.012158, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "cohere/command-a-03-2025": { "accuracy": 13.333, "latency": 23.345, "stderr": 0.945, "cost_per_test": 0.011732, "temperature": 0.3, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "openai/gpt-4o-2024-11-20": { "accuracy": 11.875, "latency": 15.776, "stderr": 0.955, "cost_per_test": 0.011954, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 11.458, "latency": 28.771, "stderr": 1.03, "cost_per_test": 0.00101, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 10.0, "latency": 9.188, "stderr": 0.938, "cost_per_test": 0.008821, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 9.375, "latency": 15.851, "stderr": 0.54, "cost_per_test": 0.0, "temperature": 0.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "mistralai/mistral-large-2411": { "accuracy": 9.167, "latency": 19.676, "stderr": 1.091, "cost_per_test": 0.006338, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/mistral-small-2402": { "accuracy": 5.625, "latency": 13.228, "stderr": 0.625, "cost_per_test": 0.000782, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/mistral-small-2503": { "accuracy": 3.542, "latency": 11.675, "stderr": 0.208, "cost_per_test": 0.000293, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 3.333, "latency": 9.055, "stderr": 0.315, "cost_per_test": 0.002432, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "ai21labs/jamba-mini-1.6": { "accuracy": 0.417, "latency": 6.624, "stderr": 0.273, "cost_per_test": 0.000456, "temperature": 0.4, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "ai21labs/jamba-large-1.6": { "accuracy": 0.417, "latency": 18.858, "stderr": 0.417, "cost_per_test": 0.007658, "temperature": 0.4, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } }, "aime_2024": { "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 99.583, "latency": 196.019, "stderr": 0.417, "cost_per_test": 1.212538, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3.1-pro-preview": { "accuracy": 99.167, "latency": 93.324, "stderr": 0.546, "cost_per_test": 0.111396, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3-pro-preview": { "accuracy": 99.167, "latency": 105.646, "stderr": 0.546, "cost_per_test": 0.162595, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 98.75, "latency": 97.839, "stderr": 0.61, "cost_per_test": 0.051914, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 97.5, "latency": 248.186, "stderr": 0.833, "cost_per_test": 0.285745, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "kimi/kimi-k2.5-thinking": { "accuracy": 97.5, "latency": 294.43, "stderr": 0.546, "cost_per_test": 0.058907, "temperature": null, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 97.083, "latency": 101.103, "stderr": 0.417, "cost_per_test": 0.120107, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.4-2026-03-05": { "accuracy": 97.083, "latency": 102.378, "stderr": 0.984, "cost_per_test": 0.117288, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "meta/muse_spark": { "accuracy": 96.667, "latency": 127.304, "stderr": 0.63, "cost_per_test": 0.003681, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta" }, "anthropic/claude-opus-4-7": { "accuracy": 96.667, "latency": 136.711, "stderr": 0.0, "cost_per_test": 0.339778, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 95.833, "latency": 290.099, "stderr": 0.546, "cost_per_test": 0.118454, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3-flash-preview": { "accuracy": 95.833, "latency": 80.8, "stderr": 0.833, "cost_per_test": 0.04424, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3.6-plus": { "accuracy": 95.417, "latency": 510.364, "stderr": 1.08, "cost_per_test": 0.092649, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "zai/glm-4.7": { "accuracy": 95.0, "latency": 502.917, "stderr": 0.63, "cost_per_test": 0.038545, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "zai/glm-4.6": { "accuracy": 94.583, "latency": 625.269, "stderr": 0.877, "cost_per_test": 0.102339, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "grok/grok-4-fast-reasoning": { "accuracy": 94.167, "latency": 31.664, "stderr": 0.833, "cost_per_test": 0.003984, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5-2025-08-07": { "accuracy": 94.167, "latency": 336.233, "stderr": 0.833, "cost_per_test": 0.085431, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 93.75, "latency": 145.315, "stderr": 0.984, "cost_per_test": 0.132848, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-0709": { "accuracy": 93.333, "latency": 103.985, "stderr": 3.333, "cost_per_test": 0.141831, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 93.333, "latency": 83.322, "stderr": 0.891, "cost_per_test": 0.00373, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "alibaba/qwen3.5-flash": { "accuracy": 93.333, "latency": 230.487, "stderr": 1.409, "cost_per_test": 0.010694, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "anthropic/claude-sonnet-4-6": { "accuracy": 93.333, "latency": 267.082, "stderr": 0.0, "cost_per_test": 0.351439, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-5-thinking": { "accuracy": 93.333, "latency": 410.326, "stderr": 1.091, "cost_per_test": 0.062435, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "fireworks/gpt-oss-120b": { "accuracy": 93.113, "latency": 162.754, "stderr": 1.908, "cost_per_test": 0.00998, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 92.917, "latency": 86.475, "stderr": 0.755, "cost_per_test": 0.020259, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "zai/glm-5.1": { "accuracy": 92.083, "latency": 339.408, "stderr": 0.61, "cost_per_test": 0.063791, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "minimax/MiniMax-M2.7": { "accuracy": 91.667, "latency": 236.514, "stderr": 1.26, "cost_per_test": 0.022761, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 90.667, "latency": 205.318, "stderr": 0.667, "cost_per_test": 0.249445, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 90.417, "latency": 17.271, "stderr": 2.13, "cost_per_test": 0.088484, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/o3-mini-2025-01-31": { "accuracy": 89.167, "latency": 175.552, "stderr": 1.045, "cost_per_test": 0.049871, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "minimax/MiniMax-M2.5": { "accuracy": 89.167, "latency": 343.683, "stderr": 1.373, "cost_per_test": 0.038321, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 88.75, "latency": 94.102, "stderr": 0.877, "cost_per_test": 0.007605, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 88.333, "latency": 38.16, "stderr": 0.891, "cost_per_test": 0.00649, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 88.333, "latency": 202.315, "stderr": 1.543, "cost_per_test": 0.042889, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 87.5, "latency": 121.572, "stderr": 1.596, "cost_per_test": 0.025552, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 87.5, "latency": 356.683, "stderr": 1.045, "cost_per_test": 0.004382, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o3-2025-04-16": { "accuracy": 87.222, "latency": 173.738, "stderr": 2.344, "cost_per_test": 0.052328, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "zai/glm-4.5": { "accuracy": 87.083, "latency": 322.751, "stderr": 1.327, "cost_per_test": 0.044344, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "mistralai/magistral-medium-2509": { "accuracy": 87.083, "latency": 374.391, "stderr": 1.598, "cost_per_test": 0.0798, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/gpt-oss-20b": { "accuracy": 86.667, "latency": 236.967, "stderr": 1.409, "cost_per_test": 0.010258, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "alibaba/qwen3-max": { "accuracy": 85.833, "latency": 15.159, "stderr": 1.22, "cost_per_test": 0.031117, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "kimi/kimi-k2-thinking": { "accuracy": 85.833, "latency": 809.713, "stderr": 2.065, "cost_per_test": 0.038294, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 85.417, "latency": 58.769, "stderr": 1.4, "cost_per_test": 0.002955, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/o4-mini-2025-04-16": { "accuracy": 84.667, "latency": 50.336, "stderr": 2.0, "cost_per_test": 0.022297, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/qwen3-235b-a22b": { "accuracy": 84.583, "latency": 227.556, "stderr": 0.61, "cost_per_test": 0.010654, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "mistralai/magistral-small-2509": { "accuracy": 84.583, "latency": 262.151, "stderr": 1.08, "cost_per_test": 0.02346, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 83.75, "latency": 152.656, "stderr": 0.755, "cost_per_test": 0.174909, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 83.441, "latency": 211.203, "stderr": 1.45, "cost_per_test": 0.845636, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 82.5, "latency": 264.28, "stderr": 1.637, "cost_per_test": 0.283075, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 81.111, "latency": 193.162, "stderr": 1.648, "cost_per_test": 0.008014, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "minimax/MiniMax-M2.1": { "accuracy": 79.167, "latency": 217.936, "stderr": 1.754, "cost_per_test": 0.022637, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "fireworks/deepseek-r1": { "accuracy": 77.5, "latency": 153.052, "stderr": 1.51, "cost_per_test": 0.048844, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 73.75, "latency": 30.443, "stderr": 0.755, "cost_per_test": 0.002841, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/o1-2024-12-17": { "accuracy": 73.333, "latency": 166.783, "stderr": 1.992, "cost_per_test": 0.634309, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 70.0, "latency": 110.392, "stderr": 1.543, "cost_per_test": 0.011401, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "fireworks/deepseek-v3p2": { "accuracy": 69.167, "latency": 120.589, "stderr": 2.938, "cost_per_test": 0.00137, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "alibaba/qwen3-max-preview": { "accuracy": 65.417, "latency": 182.155, "stderr": 3.208, "cost_per_test": 0.022332, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 61.25, "latency": 161.92, "stderr": 2.813, "cost_per_test": 0.0, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "grok/grok-3": { "accuracy": 60.417, "latency": 64.248, "stderr": 2.394, "cost_per_test": 0.092816, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-v3-0324": { "accuracy": 58.571, "latency": 49.485, "stderr": 2.804, "cost_per_test": 0.003187, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 54.583, "latency": 57.704, "stderr": 2.741, "cost_per_test": 0.005092, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/mistral-large-2512": { "accuracy": 54.583, "latency": 101.853, "stderr": 0.61, "cost_per_test": 0.005864, "temperature": 0.065, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 52.917, "latency": 30.512, "stderr": 1.83, "cost_per_test": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 52.917, "latency": 55.114, "stderr": 2.917, "cost_per_test": 0.004435, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 52.5, "latency": 44.809, "stderr": 2.578, "cost_per_test": 0.000551, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 50.833, "latency": 33.709, "stderr": 1.51, "cost_per_test": 0.008168, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-medium-2505": { "accuracy": 50.0, "latency": 68.044, "stderr": 1.89, "cost_per_test": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 49.167, "latency": 293.56, "stderr": 2.159, "cost_per_test": 0.380382, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514": { "accuracy": 48.75, "latency": 38.331, "stderr": 1.4, "cost_per_test": 0.11308, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 48.75, "latency": 245.542, "stderr": 2.435, "cost_per_test": 0.04257, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 43.333, "latency": 23.206, "stderr": 1.667, "cost_per_test": 0.021848, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 32.917, "latency": 15.429, "stderr": 1.327, "cost_per_test": 0.001131, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "google/gemini-2.0-flash-001": { "accuracy": 32.5, "latency": 12.103, "stderr": 0.833, "cost_per_test": 0.001136, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 31.25, "latency": 17.094, "stderr": 2.176, "cost_per_test": 0.002931, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/deepseek-v3": { "accuracy": 30.417, "latency": 66.28, "stderr": 1.327, "cost_per_test": 0.003775, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 28.75, "latency": 13.068, "stderr": 1.25, "cost_per_test": 0.001266, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 27.917, "latency": 21.568, "stderr": 1.08, "cost_per_test": 0.000789, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 26.25, "latency": 19.427, "stderr": 1.327, "cost_per_test": 0.023944, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 24.167, "latency": 11.446, "stderr": 1.373, "cost_per_test": 0.001148, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "google/gemini-1.5-pro-002": { "accuracy": 23.75, "latency": 10.658, "stderr": 0.755, "cost_per_test": 0.00421, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 20.833, "latency": 6.92, "stderr": 0.833, "cost_per_test": 0.00097, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 20.833, "latency": 12.902, "stderr": 2.8, "cost_per_test": 0.00059, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-2-1212": { "accuracy": 19.583, "latency": 58.028, "stderr": 1.327, "cost_per_test": 0.014342, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-1.5-flash-002": { "accuracy": 17.083, "latency": 5.826, "stderr": 1.327, "cost_per_test": 0.000271, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 15.417, "latency": 8.98, "stderr": 1.535, "cost_per_test": 0.008645, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "cohere/command-a-03-2025": { "accuracy": 15.0, "latency": 23.494, "stderr": 1.543, "cost_per_test": 0.012986, "temperature": 0.3, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "openai/gpt-4o-2024-11-20": { "accuracy": 14.167, "latency": 15.959, "stderr": 1.045, "cost_per_test": 0.011915, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 13.333, "latency": 16.094, "stderr": 0.63, "cost_per_test": 0.0, "temperature": 0.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 12.917, "latency": 101.51, "stderr": 1.469, "cost_per_test": 0.014132, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-large-2411": { "accuracy": 11.667, "latency": 21.843, "stderr": 0.891, "cost_per_test": 0.006976, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 8.75, "latency": 40.44, "stderr": 1.535, "cost_per_test": 0.001342, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-small-2402": { "accuracy": 7.917, "latency": 17.568, "stderr": 0.61, "cost_per_test": 0.001034, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 6.667, "latency": 8.758, "stderr": 0.63, "cost_per_test": 0.002344, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/mistral-small-2503": { "accuracy": 3.75, "latency": 13.125, "stderr": 0.417, "cost_per_test": 0.00034, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 0.833, "latency": 6.411, "stderr": 0.546, "cost_per_test": 0.000438, "temperature": 0.4, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "ai21labs/jamba-large-1.6": { "accuracy": 0.833, "latency": 18.75, "stderr": 0.833, "cost_per_test": 0.007623, "temperature": 0.4, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } }, "aime_2025": { "google/gemini-3.1-pro-preview": { "accuracy": 97.083, "latency": 115.726, "stderr": 0.417, "cost_per_test": 0.142586, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "meta/muse_spark": { "accuracy": 97.083, "latency": 186.187, "stderr": 0.417, "cost_per_test": 0.005441, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 96.667, "latency": 139.517, "stderr": 0.0, "cost_per_test": 0.150539, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.4-2026-03-05": { "accuracy": 96.25, "latency": 140.554, "stderr": 0.417, "cost_per_test": 0.159085, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-7": { "accuracy": 95.833, "latency": 227.418, "stderr": 1.045, "cost_per_test": 0.561248, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3-flash-preview": { "accuracy": 95.417, "latency": 87.215, "stderr": 0.61, "cost_per_test": 0.048415, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 95.417, "latency": 310.003, "stderr": 0.61, "cost_per_test": 0.138819, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3-pro-preview": { "accuracy": 94.194, "latency": 112.882, "stderr": 0.54, "cost_per_test": 0.176016, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 94.167, "latency": 115.759, "stderr": 0.833, "cost_per_test": 0.06688, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 93.75, "latency": 343.169, "stderr": 0.984, "cost_per_test": 0.401897, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "kimi/kimi-k2.5-thinking": { "accuracy": 93.75, "latency": 350.791, "stderr": 0.417, "cost_per_test": 0.069378, "temperature": null, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "alibaba/qwen3.6-plus": { "accuracy": 93.75, "latency": 522.617, "stderr": 0.417, "cost_per_test": 0.09439, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 92.917, "latency": 182.468, "stderr": 0.755, "cost_per_test": 0.16786, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5-2025-08-07": { "accuracy": 92.581, "latency": 248.171, "stderr": 2.474, "cost_per_test": 0.099955, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 92.083, "latency": 72.984, "stderr": 1.4, "cost_per_test": 0.003908, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/gpt-oss-120b": { "accuracy": 92.083, "latency": 150.504, "stderr": 1.535, "cost_per_test": 0.010965, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "alibaba/qwen3.5-flash": { "accuracy": 91.667, "latency": 248.72, "stderr": 1.091, "cost_per_test": 0.01154, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "zai/glm-5.1": { "accuracy": 91.667, "latency": 410.994, "stderr": 0.63, "cost_per_test": 0.078105, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "zai/glm-4.7": { "accuracy": 91.667, "latency": 555.468, "stderr": 0.63, "cost_per_test": 0.048071, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 91.25, "latency": 228.383, "stderr": 0.61, "cost_per_test": 1.427953, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-sonnet-4-6": { "accuracy": 91.25, "latency": 388.987, "stderr": 0.877, "cost_per_test": 0.520585, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-4.6": { "accuracy": 90.833, "latency": 774.468, "stderr": 0.833, "cost_per_test": 0.130232, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 90.417, "latency": 100.673, "stderr": 0.755, "cost_per_test": 0.004458, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "minimax/MiniMax-M2.7": { "accuracy": 90.417, "latency": 236.759, "stderr": 0.755, "cost_per_test": 0.023157, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 90.0, "latency": 124.585, "stderr": 1.491, "cost_per_test": 0.025261, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI" }, "zai/glm-5-thinking": { "accuracy": 90.0, "latency": 507.062, "stderr": 1.091, "cost_per_test": 0.077199, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "grok/grok-4-fast-reasoning": { "accuracy": 88.333, "latency": 37.454, "stderr": 1.89, "cost_per_test": 0.00465, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "minimax/MiniMax-M2.5": { "accuracy": 88.333, "latency": 403.292, "stderr": 1.091, "cost_per_test": 0.044437, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "grok/grok-4-0709": { "accuracy": 87.778, "latency": 162.471, "stderr": 1.111, "cost_per_test": 0.206967, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "zai/glm-4.5": { "accuracy": 86.25, "latency": 343.484, "stderr": 2.309, "cost_per_test": 0.049437, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 85.714, "latency": 229.69, "stderr": 1.198, "cost_per_test": 0.284409, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/gpt-oss-20b": { "accuracy": 85.417, "latency": 252.457, "stderr": 1.775, "cost_per_test": 0.011828, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "kimi/kimi-k2-thinking": { "accuracy": 85.0, "latency": 1040.114, "stderr": 1.667, "cost_per_test": 0.050722, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 84.167, "latency": 166.242, "stderr": 2.097, "cost_per_test": 0.028726, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/o3-mini-2025-01-31": { "accuracy": 83.75, "latency": 133.747, "stderr": 1.83, "cost_per_test": 0.051763, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 83.75, "latency": 224.758, "stderr": 1.598, "cost_per_test": 0.047431, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "fireworks/qwen3-235b-a22b": { "accuracy": 83.333, "latency": 256.792, "stderr": 1.409, "cost_per_test": 0.012542, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/o3-2025-04-16": { "accuracy": 83.333, "latency": 358.62, "stderr": 1.491, "cost_per_test": 0.06787, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/o4-mini-2025-04-16": { "accuracy": 82.667, "latency": 59.883, "stderr": 1.247, "cost_per_test": 0.029117, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 81.667, "latency": 163.843, "stderr": 1.667, "cost_per_test": 0.172641, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 81.667, "latency": 389.462, "stderr": 1.992, "cost_per_test": 0.004458, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 81.25, "latency": 110.39, "stderr": 0.61, "cost_per_test": 0.008957, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 81.25, "latency": 194.202, "stderr": 1.986, "cost_per_test": 0.00971, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/magistral-medium-2509": { "accuracy": 80.0, "latency": 427.624, "stderr": 1.26, "cost_per_test": 0.094015, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 78.333, "latency": 42.968, "stderr": 2.182, "cost_per_test": 0.007442, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/magistral-small-2509": { "accuracy": 76.774, "latency": 273.812, "stderr": 2.161, "cost_per_test": 0.026765, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "minimax/MiniMax-M2.1": { "accuracy": 76.667, "latency": 257.333, "stderr": 1.89, "cost_per_test": 0.025666, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "alibaba/qwen3-max": { "accuracy": 76.25, "latency": 18.362, "stderr": 1.83, "cost_per_test": 0.035612, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 72.917, "latency": 217.805, "stderr": 2.917, "cost_per_test": 0.874244, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-r1": { "accuracy": 70.417, "latency": 154.772, "stderr": 2.222, "cost_per_test": 0.057942, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 70.0, "latency": 279.296, "stderr": 2.357, "cost_per_test": 0.306977, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/o1-2024-12-17": { "accuracy": 69.583, "latency": 187.272, "stderr": 2.13, "cost_per_test": 0.683645, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 67.5, "latency": 32.352, "stderr": 0.833, "cost_per_test": 0.00302, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 63.333, "latency": 17.614, "stderr": 1.782, "cost_per_test": 0.090022, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3p2": { "accuracy": 60.417, "latency": 118.521, "stderr": 2.848, "cost_per_test": 0.001428, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-3": { "accuracy": 57.083, "latency": 63.73, "stderr": 1.598, "cost_per_test": 0.088521, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "alibaba/qwen3-max-preview": { "accuracy": 55.972, "latency": 231.351, "stderr": 3.995, "cost_per_test": 0.023777, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 55.417, "latency": 138.944, "stderr": 1.883, "cost_per_test": 0.01416, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 48.333, "latency": 65.497, "stderr": 2.746, "cost_per_test": 0.004879, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 47.917, "latency": 32.569, "stderr": 1.659, "cost_per_test": 0.007633, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 46.667, "latency": 61.642, "stderr": 3.333, "cost_per_test": 0.004614, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4-fast-non-reasoning": { "accuracy": 45.833, "latency": 4.837, "stderr": 1.967, "cost_per_test": 0.00064, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "fireworks/deepseek-v3-0324": { "accuracy": 45.833, "latency": 51.665, "stderr": 1.967, "cost_per_test": 0.003198, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 45.833, "latency": 173.867, "stderr": 1.51, "cost_per_test": 0.0, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 40.0, "latency": 313.858, "stderr": 2.746, "cost_per_test": 0.410312, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 35.556, "latency": 28.838, "stderr": 2.811, "cost_per_test": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/mistral-medium-2505": { "accuracy": 34.583, "latency": 63.864, "stderr": 1.883, "cost_per_test": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 33.75, "latency": 23.59, "stderr": 0.984, "cost_per_test": 0.021431, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-20250514": { "accuracy": 33.75, "latency": 35.727, "stderr": 1.83, "cost_per_test": 0.107687, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 33.333, "latency": 21.828, "stderr": 2.673, "cost_per_test": 0.000987, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 31.667, "latency": 57.484, "stderr": 1.782, "cost_per_test": 0.000582, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/mistral-large-2512": { "accuracy": 31.25, "latency": 106.797, "stderr": 1.659, "cost_per_test": 0.006307, "temperature": 0.065, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "openai/gpt-4.1-2025-04-14": { "accuracy": 30.417, "latency": 76.609, "stderr": 1.327, "cost_per_test": 0.057641, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.0-flash-001": { "accuracy": 27.083, "latency": 10.326, "stderr": 0.984, "cost_per_test": 0.000951, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/deepseek-v3": { "accuracy": 24.583, "latency": 51.322, "stderr": 0.61, "cost_per_test": 0.002953, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 24.167, "latency": 10.759, "stderr": 1.045, "cost_per_test": 0.001103, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 21.25, "latency": 20.313, "stderr": 3.33, "cost_per_test": 0.003516, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 18.333, "latency": 18.429, "stderr": 2.357, "cost_per_test": 0.022458, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-1.5-flash-002": { "accuracy": 17.5, "latency": 5.567, "stderr": 1.22, "cost_per_test": 0.00026, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 17.5, "latency": 15.575, "stderr": 0.833, "cost_per_test": 0.001154, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "openai/gpt-4o-2024-08-06": { "accuracy": 15.0, "latency": 35.226, "stderr": 1.26, "cost_per_test": 0.010184, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 14.167, "latency": 17.102, "stderr": 1.373, "cost_per_test": 0.000678, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-1.5-pro-002": { "accuracy": 13.75, "latency": 10.626, "stderr": 0.755, "cost_per_test": 0.004196, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "cohere/command-a-03-2025": { "accuracy": 11.667, "latency": 23.196, "stderr": 1.091, "cost_per_test": 0.010479, "temperature": 0.3, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere" }, "grok/grok-2-1212": { "accuracy": 10.833, "latency": 57.739, "stderr": 0.833, "cost_per_test": 0.015694, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 10.0, "latency": 21.813, "stderr": 0.891, "cost_per_test": 0.000837, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "openai/gpt-4o-2024-11-20": { "accuracy": 9.583, "latency": 15.593, "stderr": 1.598, "cost_per_test": 0.011992, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 7.917, "latency": 11.042, "stderr": 1.659, "cost_per_test": 0.001123, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "mistralai/mistral-large-2411": { "accuracy": 6.667, "latency": 17.509, "stderr": 1.992, "cost_per_test": 0.0057, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 5.417, "latency": 15.609, "stderr": 0.877, "cost_per_test": 0.0, "temperature": 0.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI" }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 4.583, "latency": 9.395, "stderr": 1.08, "cost_per_test": 0.008997, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "mistralai/mistral-small-2402": { "accuracy": 3.333, "latency": 8.888, "stderr": 1.091, "cost_per_test": 0.000531, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "mistralai/mistral-small-2503": { "accuracy": 3.333, "latency": 10.225, "stderr": 0.0, "cost_per_test": 0.000246, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "ai21labs/jamba-mini-1.6": { "accuracy": 0.0, "latency": 6.837, "stderr": 0.0, "cost_per_test": 0.000473, "temperature": 0.4, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 0.0, "latency": 9.352, "stderr": 0.0, "cost_per_test": 0.00252, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "ai21labs/jamba-large-1.6": { "accuracy": 0.0, "latency": 18.965, "stderr": 0.0, "cost_per_test": 0.007693, "temperature": 0.4, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs" } } } }