{ "metadata": { "benchmark": "GPQA Diamond", "slug": "gpqa", "description": "Graduate-level Google-Proof Q&A benchmark evaluating models on questions that require deep reasoning.", "benchmark_id": "gpqa", "family": "gpqa", "version": "1", "updated": "2026-09-01", "dataset_type": "public", "industry": "academic", "tasks": { "overall": "Overall", "diamond_few_shot_cot": "Few-Shot CoT", "diamond_zero_shot_cot": "Zero-Shot CoT" }, "models": [ "ai21labs/jamba-large-1.6", "ai21labs/jamba-mini-1.6", "alibaba/qwen3-max", "alibaba/qwen3-max-2026-01-23", "alibaba/qwen3-max-preview", "alibaba/qwen3.5-flash", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "ant/ling-3.0-flash-2607", "anthropic/claude-3-5-haiku-20241022", "anthropic/claude-3-5-sonnet-20241022", "anthropic/claude-3-7-sonnet-20250219", "anthropic/claude-3-7-sonnet-20250219-thinking", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-1-20250805-thinking", "anthropic/claude-opus-4-20250514", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "cohere/command-a-03-2025", "cohere/command-r-plus", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "fireworks/deepseek-v3", "fireworks/deepseek-v3-0324", "fireworks/deepseek-v3p2", "fireworks/deepseek-v3p2-thinking", "fireworks/gpt-oss-120b", "fireworks/gpt-oss-20b", "fireworks/llama4-maverick-instruct-basic", "fireworks/qwen3-235b-a22b", "google/gemini-1.5-flash-002", "google/gemini-1.5-pro-002", "google/gemini-2.0-flash-001", "google/gemini-2.5-flash-lite-preview-09-2025", "google/gemini-2.5-flash-lite-preview-09-2025-thinking", "google/gemini-2.5-flash-preview-04-17", "google/gemini-2.5-flash-preview-09-2025", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-pro-exp-03-25", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "grok/grok-2-1212", "grok/grok-3", "grok/grok-3-mini-fast-high-reasoning", "grok/grok-3-mini-fast-low-reasoning", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "kimi/kimi-k2-thinking", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse_spark", "meta/muse_spark_1_1", "minimax/MiniMax-M2.1", "minimax/MiniMax-M2.5", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/magistral-medium-2509", "mistralai/magistral-small-2509", "mistralai/mistral-large-2411", "mistralai/mistral-large-2512", "mistralai/mistral-medium-3.5", "mistralai/mistral-small-2402", "mistralai/mistral-small-2503", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-3.5-turbo", "openai/gpt-4.1-2025-04-14", "openai/gpt-4.1-mini-2025-04-14", "openai/gpt-4.1-nano-2025-04-14", "openai/gpt-4o-2024-05-13", "openai/gpt-4o-2024-11-20", "openai/gpt-4o-mini-2024-07-18", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-nano-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/o1-2024-12-17", "openai/o3-2025-04-16", "openai/o3-mini-2025-01-31", "openai/o4-mini-2025-04-16", "poolside/laguna-m.1", "poolside/laguna-xs.2", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561", "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking", "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "together/meta-llama/Llama-4-Scout-17B-16E-Instruct", "together/moonshotai/Kimi-K2-Instruct", "xiaomi/mimo-v2-flash", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "zai/glm-4.5", "zai/glm-4.6", "zai/glm-4.7", "zai/glm-5-thinking", "zai/glm-5.1", "zai/glm-5.2", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "platform", "mode": "one-shot", "archived": false, "partner": false, "total_models": 138 }, "tasks": { "overall": { "google/gemini-3.1-pro-preview": { "accuracy": 95.454, "latency": 65.76, "stderr": 1.046, "cost_per_test": 0.081442, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 95.202, "latency": 58.304, "stderr": 1.074, "cost_per_test": 0.10996, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 94.697, "latency": 209.056, "stderr": 1.125, "cost_per_test": 0.045365, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 94.444, "latency": 20.18, "stderr": 1.477, "cost_per_test": 0.056302, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 93.94, "latency": 8.773, "stderr": 1.488, "cost_per_test": 0.023122, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 93.686, "latency": 266.606, "stderr": 1.221, "cost_per_test": 0.077028, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 93.434, "latency": 18.064, "stderr": 1.329, "cost_per_test": 0.03189, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 93.434, "latency": 39.588, "stderr": 1.244, "cost_per_test": 0.095481, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 93.434, "latency": 45.857, "stderr": 1.863, "cost_per_test": 0.204523, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 93.182, "latency": 99.902, "stderr": 1.939, "cost_per_test": 0.450473, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 93.182, "latency": 112.326, "stderr": 1.288, "cost_per_test": 0.082767, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 92.929, "latency": 122.425, "stderr": 1.309, "cost_per_test": 0.088491, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.5": { "accuracy": 92.929, "latency": 217.858, "stderr": 1.288, "cost_per_test": 0.044029, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 92.676, "latency": 27.655, "stderr": 1.462, "cost_per_test": 0.063842, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 92.676, "latency": 278.908, "stderr": 1.443, "cost_per_test": 0.0255, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 92.424, "latency": 133.876, "stderr": 1.984, "cost_per_test": 0.339261, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 92.424, "latency": 279.412, "stderr": 2.02, "cost_per_test": 0.065819, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 91.666, "latency": 53.521, "stderr": 1.736, "cost_per_test": 0.010616, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 91.666, "latency": 61.34, "stderr": 1.389, "cost_per_test": 0.089573, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 91.666, "latency": 173.214, "stderr": 1.837, "cost_per_test": 0.149037, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 91.666, "latency": 202.953, "stderr": 1.908, "cost_per_test": 0.20934, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 91.414, "latency": 739.636, "stderr": 1.406, "cost_per_test": 0.025366, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 91.162, "latency": 75.198, "stderr": 2.104, "cost_per_test": 0.034304, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 90.909, "latency": 36.764, "stderr": 1.906, "cost_per_test": 0.033997, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 90.152, "latency": 126.611, "stderr": 1.497, "cost_per_test": 0.06149, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 90.152, "latency": 202.52, "stderr": 1.545, "cost_per_test": 0.365129, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 89.899, "latency": 79.956, "stderr": 1.694, "cost_per_test": 0.012875, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark": { "accuracy": 89.646, "latency": 94.58, "stderr": 1.546, "cost_per_test": 0.003074, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 89.646, "latency": 319.02, "stderr": 1.186, "cost_per_test": 0.294901, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 89.394, "latency": 900.378, "stderr": 1.654, "cost_per_test": 0.125847, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 89.142, "latency": 544.744, "stderr": 1.997, "cost_per_test": 0.080129, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 88.889, "latency": 63.234, "stderr": 2.224, "cost_per_test": 0.077709, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 88.889, "latency": 116.361, "stderr": 1.579, "cost_per_test": 0.046624, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 88.636, "latency": 59.421, "stderr": 1.594, "cost_per_test": 0.034461, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-0709": { "accuracy": 88.132, "latency": 115.516, "stderr": 1.625, "cost_per_test": 0.147971, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 88.132, "latency": 210.7, "stderr": 1.851, "cost_per_test": 0.055878, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 87.879, "latency": 83.84, "stderr": 1.638, "cost_per_test": 0.044346, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 87.374, "latency": 103.772, "stderr": 1.669, "cost_per_test": 0.018542, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 87.374, "latency": 122.081, "stderr": 1.668, "cost_per_test": 0.022964, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 87.122, "latency": 308.595, "stderr": 1.784, "cost_per_test": 0.16589, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 86.616, "latency": 85.708, "stderr": 1.711, "cost_per_test": 0.074563, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 86.616, "latency": 171.727, "stderr": 1.953, "cost_per_test": 0.014051, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 86.364, "latency": 131.004, "stderr": 2.126, "cost_per_test": 0.001626, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 86.112, "latency": 158.471, "stderr": 1.738, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 85.859, "latency": 138.525, "stderr": 2.476, "cost_per_test": 0.712758, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 85.606, "latency": 169.716, "stderr": 1.764, "cost_per_test": 0.072708, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 85.606, "latency": 224.664, "stderr": 1.774, "cost_per_test": 0.05711, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 85.606, "latency": 377.864, "stderr": 2.406, "cost_per_test": 0.408666, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 85.354, "latency": 21.408, "stderr": 1.789, "cost_per_test": 0.003049, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 84.848, "latency": 43.513, "stderr": 2.197, "cost_per_test": 0.003327, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 84.848, "latency": 368.073, "stderr": 1.799, "cost_per_test": 0.0741, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.1": { "accuracy": 84.518, "latency": 249.704, "stderr": 1.821, "cost_per_test": 0.044897, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 84.343, "latency": 68.481, "stderr": 1.85, "cost_per_test": 0.002916, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 84.091, "latency": 65.777, "stderr": 1.861, "cost_per_test": 0.035552, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 84.091, "latency": 151.843, "stderr": 2.135, "cost_per_test": 0.029962, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 83.838, "latency": 13.848, "stderr": 1.847, "cost_per_test": 0.01235, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 83.586, "latency": 259.758, "stderr": 2.036, "cost_per_test": 0.020022, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5-thinking": { "accuracy": 83.333, "latency": 242.523, "stderr": 1.873, "cost_per_test": 0.034091, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 83.08, "latency": 63.11, "stderr": 2.457, "cost_per_test": 0.012743, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 82.828, "latency": 79.884, "stderr": 1.895, "cost_per_test": 0.003413, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 82.576, "latency": 251.332, "stderr": 2.468, "cost_per_test": 0.006654, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 82.07, "latency": 171.962, "stderr": 2.258, "cost_per_test": 0.017869, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 81.633, "latency": 117.251, "stderr": 2.254, "cost_per_test": 0.114262, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 81.566, "latency": 19.483, "stderr": 1.996, "cost_per_test": 0.002894, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 81.566, "latency": 89.393, "stderr": 2.054, "cost_per_test": 0.002538, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 81.06, "latency": 16.464, "stderr": 1.967, "cost_per_test": 0.002298, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 80.808, "latency": 41.102, "stderr": 1.998, "cost_per_test": 0.012944, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 80.303, "latency": 67.133, "stderr": 1.996, "cost_per_test": 0.013593, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 80.303, "latency": 260.248, "stderr": 1.997, "cost_per_test": 0.003465, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-4.7": { "accuracy": 80.05, "latency": 211.079, "stderr": 2.006, "cost_per_test": 0.024866, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 79.546, "latency": 13.54, "stderr": 2.463, "cost_per_test": 0.074932, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "alibaba/qwen3-max": { "accuracy": 79.546, "latency": 70.152, "stderr": 2.026, "cost_per_test": 0.016466, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 79.293, "latency": 40.618, "stderr": 2.044, "cost_per_test": 0.00388, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 78.536, "latency": 56.176, "stderr": 2.253, "cost_per_test": 0.006212, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 78.536, "latency": 135.412, "stderr": 2.128, "cost_per_test": 0.011422, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 78.536, "latency": 585.555, "stderr": 2.183, "cost_per_test": 0.023072, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 77.778, "latency": 106.784, "stderr": 2.09, "cost_per_test": 0.01338, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 77.526, "latency": 29.498, "stderr": 2.417, "cost_per_test": 0.001565, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 76.516, "latency": 23.53, "stderr": 2.154, "cost_per_test": 0.002833, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 76.263, "latency": 35.772, "stderr": 2.16, "cost_per_test": 0.144678, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 76.263, "latency": 46.828, "stderr": 2.161, "cost_per_test": 0.00065, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 75.505, "latency": 99.984, "stderr": 2.176, "cost_per_test": 0.035857, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 75.253, "latency": 155.02, "stderr": 2.169, "cost_per_test": 0.18875, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 75.0, "latency": 118.458, "stderr": 2.191, "cost_per_test": 0.116706, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 74.495, "latency": 30.004, "stderr": 2.19, "cost_per_test": 0.012961, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.6": { "accuracy": 74.495, "latency": 474.834, "stderr": 2.197, "cost_per_test": 0.01541, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-3": { "accuracy": 74.242, "latency": 29.912, "stderr": 2.209, "cost_per_test": 0.039242, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 73.232, "latency": 40.176, "stderr": 2.231, "cost_per_test": 0.252632, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 72.98, "latency": 14.234, "stderr": 2.233, "cost_per_test": 0.001599, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 72.222, "latency": 83.203, "stderr": 2.474, "cost_per_test": 0.055064, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.5": { "accuracy": 72.222, "latency": 704.224, "stderr": 2.262, "cost_per_test": 0.019454, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 71.717, "latency": 19.13, "stderr": 2.261, "cost_per_test": 0.066808, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 71.464, "latency": 41.434, "stderr": 2.266, "cost_per_test": 0.004481, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 70.202, "latency": 20.61, "stderr": 2.298, "cost_per_test": 0.000481, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 70.202, "latency": 306.218, "stderr": 2.373, "cost_per_test": 0.006472, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 69.95, "latency": 35.934, "stderr": 2.303, "cost_per_test": 0.067421, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 69.444, "latency": 10.071, "stderr": 2.35, "cost_per_test": 0.000964, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 69.444, "latency": 12.818, "stderr": 2.315, "cost_per_test": 0.013552, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 68.94, "latency": 145.092, "stderr": 2.465, "cost_per_test": 0.006738, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 68.434, "latency": 21.216, "stderr": 2.372, "cost_per_test": 0.00215, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 67.929, "latency": 9.282, "stderr": 2.351, "cost_per_test": 0.002324, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 67.424, "latency": 9.218, "stderr": 2.354, "cost_per_test": 0.011972, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 65.404, "latency": 23.14, "stderr": 2.402, "cost_per_test": 0.013852, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 65.152, "latency": 5.803, "stderr": 2.394, "cost_per_test": 0.00049, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 65.152, "latency": 8.66, "stderr": 2.398, "cost_per_test": 0.000445, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 64.142, "latency": 7.274, "stderr": 2.412, "cost_per_test": 0.001236, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 63.384, "latency": 49.716, "stderr": 2.455, "cost_per_test": 0.004578, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 62.374, "latency": 77.589, "stderr": 2.451, "cost_per_test": null, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 62.374, "latency": 94.145, "stderr": 2.435, "cost_per_test": 0.03601, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 62.122, "latency": 3.668, "stderr": 2.429, "cost_per_test": 0.001098, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 61.616, "latency": 23.475, "stderr": 2.447, "cost_per_test": 0.001552, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 59.344, "latency": 7.241, "stderr": 2.468, "cost_per_test": 0.009078, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2-flash": { "accuracy": 59.344, "latency": 20.902, "stderr": 2.496, "cost_per_test": 0.001212, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 58.334, "latency": 7.709, "stderr": 2.475, "cost_per_test": 0.003655, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 58.334, "latency": 46.252, "stderr": 2.487, "cost_per_test": 0.007058, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 57.576, "latency": 43.452, "stderr": 2.411, "cost_per_test": 0.030117, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 55.05, "latency": 69.312, "stderr": 2.499, "cost_per_test": 0.001373, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 54.546, "latency": 21.282, "stderr": 2.504, "cost_per_test": 0.001468, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 53.788, "latency": 14.924, "stderr": 2.507, "cost_per_test": 0.009465, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 50.758, "latency": 4.192, "stderr": 2.457, "cost_per_test": 0.000488, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 50.758, "latency": 7.986, "stderr": 2.495, "cost_per_test": 0.000586, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-2-1212": { "accuracy": 50.758, "latency": 20.178, "stderr": 2.51, "cost_per_test": 0.0095, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4o-2024-05-13": { "accuracy": 50.252, "latency": 10.078, "stderr": 2.512, "cost_per_test": 0.01544, "temperature": 1.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 50.0, "latency": 7.342, "stderr": 2.512, "cost_per_test": 0.001608, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 48.485, "latency": 11.49, "stderr": 2.276, "cost_per_test": 0.010094, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 47.728, "latency": 15.802, "stderr": 2.5, "cost_per_test": 0.006714, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 47.727, "latency": 16.482, "stderr": 2.468, "cost_per_test": null, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 46.97, "latency": 12.092, "stderr": 2.491, "cost_per_test": 0.000654, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 45.96, "latency": 3.39, "stderr": 2.504, "cost_per_test": 0.000233, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 44.192, "latency": 8.824, "stderr": 2.473, "cost_per_test": 0.000278, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 44.192, "latency": 15.32, "stderr": 2.495, "cost_per_test": 0.000536, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 37.879, "latency": 6.56, "stderr": 2.436, "cost_per_test": 0.00222, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 36.111, "latency": 15.471, "stderr": 2.099, "cost_per_test": 0.00669, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 34.848, "latency": 256.17, "stderr": 2.39, "cost_per_test": 0.095489, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 32.07, "latency": 4.953, "stderr": 1.988, "cost_per_test": 0.000487, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "cohere/command-r-plus": { "accuracy": 31.06, "latency": 8.194, "stderr": 2.28, "cost_per_test": 0.007438, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "openai/gpt-3.5-turbo": { "accuracy": 30.556, "latency": 2.628, "stderr": 2.287, "cost_per_test": 0.000862, "temperature": 1.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 27.02, "latency": 327.957, "stderr": 1.922, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } }, "diamond_few_shot_cot": { "google/gemini-3.1-pro-preview": { "accuracy": 95.96, "latency": 67.495, "stderr": 1.399, "cost_per_test": 0.085026, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 95.455, "latency": 9.186, "stderr": 1.76, "cost_per_test": 0.025355, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 94.949, "latency": 21.842, "stderr": 1.881, "cost_per_test": 0.061477, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 94.949, "latency": 72.403, "stderr": 1.556, "cost_per_test": 0.135633, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 93.939, "latency": 211.173, "stderr": 1.696, "cost_per_test": 0.046346, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 93.434, "latency": 118.4, "stderr": 1.822, "cost_per_test": 0.092359, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 93.434, "latency": 129.475, "stderr": 1.76, "cost_per_test": 0.093316, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 92.929, "latency": 47.631, "stderr": 1.822, "cost_per_test": 0.22434, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 92.929, "latency": 132.309, "stderr": 2.043, "cost_per_test": 0.610919, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 92.929, "latency": 288.128, "stderr": 1.822, "cost_per_test": 0.046385, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.3": { "accuracy": 92.929, "latency": 396.598, "stderr": 1.822, "cost_per_test": 0.024086, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 92.424, "latency": 16.872, "stderr": 1.991, "cost_per_test": 0.031197, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 92.424, "latency": 38.585, "stderr": 1.881, "cost_per_test": 0.100176, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 92.424, "latency": 45.557, "stderr": 2.319, "cost_per_test": 0.009378, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 92.424, "latency": 244.029, "stderr": 1.881, "cost_per_test": 0.023, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 92.424, "latency": 265.896, "stderr": 1.881, "cost_per_test": 0.077068, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 91.919, "latency": 131.379, "stderr": 2.277, "cost_per_test": 0.3381, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 91.919, "latency": 170.268, "stderr": 2.513, "cost_per_test": 0.149071, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 91.919, "latency": 278.283, "stderr": 3.024, "cost_per_test": 0.061962, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 91.414, "latency": 26.753, "stderr": 1.991, "cost_per_test": 0.066225, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 91.414, "latency": 186.909, "stderr": 3.127, "cost_per_test": 0.196199, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 90.909, "latency": 58.765, "stderr": 2.043, "cost_per_test": 0.087749, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 90.909, "latency": 203.874, "stderr": 2.043, "cost_per_test": 0.352728, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 90.404, "latency": 67.934, "stderr": 2.616, "cost_per_test": 0.030784, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 90.404, "latency": 823.036, "stderr": 2.277, "cost_per_test": 0.112916, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 89.899, "latency": 43.438, "stderr": 2.711, "cost_per_test": 0.033269, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 89.899, "latency": 80.111, "stderr": 2.233, "cost_per_test": 0.012966, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 89.899, "latency": 82.709, "stderr": 2.142, "cost_per_test": 0.044792, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 89.899, "latency": 500.4, "stderr": 2.277, "cost_per_test": 0.075267, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 88.889, "latency": 103.08, "stderr": 2.233, "cost_per_test": 0.044773, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 88.889, "latency": 205.365, "stderr": 2.36, "cost_per_test": 0.058921, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark": { "accuracy": 88.384, "latency": 94.376, "stderr": 2.277, "cost_per_test": 0.003188, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 88.384, "latency": 98.941, "stderr": 2.277, "cost_per_test": 0.017758, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 88.384, "latency": 337.27, "stderr": 1.115, "cost_per_test": 0.310954, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 87.879, "latency": 119.018, "stderr": 2.319, "cost_per_test": 0.017962, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-0709": { "accuracy": 87.879, "latency": 119.477, "stderr": 2.319, "cost_per_test": 0.152299, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 87.879, "latency": 281.977, "stderr": 2.319, "cost_per_test": 0.152118, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 87.374, "latency": 55.38, "stderr": 3.456, "cost_per_test": 0.084904, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 87.374, "latency": 60.125, "stderr": 2.36, "cost_per_test": 0.035734, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 86.364, "latency": 83.303, "stderr": 2.439, "cost_per_test": 0.077042, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 86.364, "latency": 85.167, "stderr": 2.439, "cost_per_test": 0.04534, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 86.364, "latency": 139.968, "stderr": 3.501, "cost_per_test": 0.733867, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 86.364, "latency": 142.062, "stderr": 2.439, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 85.859, "latency": 364.556, "stderr": 3.397, "cost_per_test": 0.398667, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 85.787, "latency": 197.756, "stderr": 2.488, "cost_per_test": 0.034962, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 85.354, "latency": 38.322, "stderr": 3.296, "cost_per_test": 0.002799, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 85.354, "latency": 117.31, "stderr": 2.799, "cost_per_test": 0.001594, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 85.354, "latency": 145.346, "stderr": 2.93, "cost_per_test": 0.01282, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 84.848, "latency": 60.078, "stderr": 3.55, "cost_per_test": 0.012277, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 84.848, "latency": 191.702, "stderr": 2.548, "cost_per_test": 0.074134, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 84.343, "latency": 22.395, "stderr": 2.583, "cost_per_test": 0.003746, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 84.343, "latency": 163.291, "stderr": 3.375, "cost_per_test": 0.017314, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 83.838, "latency": 70.139, "stderr": 2.616, "cost_per_test": 0.003089, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.2": { "accuracy": 83.333, "latency": 178.184, "stderr": 2.649, "cost_per_test": 0.049779, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 82.828, "latency": 74.323, "stderr": 2.68, "cost_per_test": 0.037891, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 82.828, "latency": 122.006, "stderr": 2.68, "cost_per_test": 0.024807, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 82.828, "latency": 234.216, "stderr": 2.68, "cost_per_test": 0.031964, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 82.828, "latency": 351.668, "stderr": 2.68, "cost_per_test": 0.07229, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 82.323, "latency": 19.829, "stderr": 2.711, "cost_per_test": 0.002954, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 82.323, "latency": 77.318, "stderr": 2.711, "cost_per_test": 0.003293, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 81.818, "latency": 13.94, "stderr": 2.741, "cost_per_test": 0.012617, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 81.633, "latency": 108.446, "stderr": 3.363, "cost_per_test": 0.108671, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 80.808, "latency": 237.64, "stderr": 3.429, "cost_per_test": 0.006645, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 80.303, "latency": 39.776, "stderr": 2.853, "cost_per_test": 0.014206, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 80.303, "latency": 40.258, "stderr": 2.826, "cost_per_test": 0.004123, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 79.798, "latency": 77.418, "stderr": 2.905, "cost_per_test": 0.002294, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 79.798, "latency": 241.348, "stderr": 2.905, "cost_per_test": 0.018587, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 79.293, "latency": 14.455, "stderr": 3.494, "cost_per_test": 0.092902, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 79.293, "latency": 16.085, "stderr": 2.88, "cost_per_test": 0.002329, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max": { "accuracy": 78.788, "latency": 68.635, "stderr": 2.905, "cost_per_test": 0.017168, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 78.788, "latency": 246.265, "stderr": 2.905, "cost_per_test": 0.003266, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 78.788, "latency": 546.998, "stderr": 3.127, "cost_per_test": 0.022555, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 78.283, "latency": 39.938, "stderr": 2.93, "cost_per_test": 0.014157, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.7": { "accuracy": 78.283, "latency": 205.538, "stderr": 2.93, "cost_per_test": 0.024294, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 77.778, "latency": 123.747, "stderr": 2.955, "cost_per_test": 0.014168, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 76.768, "latency": 31.524, "stderr": 3.408, "cost_per_test": 0.001658, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 76.768, "latency": 36.604, "stderr": 3.001, "cost_per_test": 0.16189, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 76.263, "latency": 41.601, "stderr": 3.088, "cost_per_test": 0.000597, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 75.758, "latency": 23.633, "stderr": 3.046, "cost_per_test": 0.002925, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 75.758, "latency": 46.429, "stderr": 3.487, "cost_per_test": 0.005601, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 75.758, "latency": 110.423, "stderr": 3.088, "cost_per_test": 0.009533, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 75.253, "latency": 139.726, "stderr": 3.067, "cost_per_test": 0.172742, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 74.747, "latency": 112.167, "stderr": 3.088, "cost_per_test": 0.112814, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 74.242, "latency": 91.417, "stderr": 3.108, "cost_per_test": 0.036567, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 73.737, "latency": 20.274, "stderr": 3.127, "cost_per_test": 0.083302, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 73.737, "latency": 28.254, "stderr": 3.127, "cost_per_test": 0.012662, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.6": { "accuracy": 73.737, "latency": 494.128, "stderr": 3.146, "cost_per_test": 0.015371, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 73.232, "latency": 75.421, "stderr": 3.542, "cost_per_test": 0.050791, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 72.727, "latency": 35.372, "stderr": 3.165, "cost_per_test": 0.227864, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3": { "accuracy": 71.717, "latency": 25.907, "stderr": 3.201, "cost_per_test": 0.0327, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 71.212, "latency": 38.525, "stderr": 3.218, "cost_per_test": 0.083439, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.5": { "accuracy": 70.707, "latency": 694.557, "stderr": 3.25, "cost_per_test": 0.020131, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 69.697, "latency": 14.615, "stderr": 3.266, "cost_per_test": 0.001837, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 69.697, "latency": 21.398, "stderr": 3.266, "cost_per_test": 0.000537, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 69.192, "latency": 13.094, "stderr": 3.281, "cost_per_test": 0.016271, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 69.192, "latency": 30.413, "stderr": 3.281, "cost_per_test": 0.004435, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 68.687, "latency": 9.856, "stderr": 3.296, "cost_per_test": 0.015357, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 68.182, "latency": 9.54, "stderr": 3.31, "cost_per_test": 0.001096, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 68.182, "latency": 279.853, "stderr": 3.375, "cost_per_test": 0.005974, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 67.172, "latency": 4.507, "stderr": 3.337, "cost_per_test": 0.001771, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 66.162, "latency": 4.976, "stderr": 3.363, "cost_per_test": 0.000512, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 65.152, "latency": 120.357, "stderr": 3.553, "cost_per_test": 0.00604, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 64.646, "latency": 8.363, "stderr": 3.397, "cost_per_test": 0.000498, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 64.646, "latency": 8.974, "stderr": 3.397, "cost_per_test": 0.002537, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 64.646, "latency": 48.056, "stderr": 3.456, "cost_per_test": 0.03391, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 64.646, "latency": 66.301, "stderr": 3.419, "cost_per_test": null, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 64.141, "latency": 20.444, "stderr": 3.419, "cost_per_test": 0.017619, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 64.141, "latency": 21.802, "stderr": 3.324, "cost_per_test": 0.002625, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 64.141, "latency": 44.156, "stderr": 3.408, "cost_per_test": 0.004136, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 62.626, "latency": 83.642, "stderr": 3.523, "cost_per_test": 0.033256, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 62.121, "latency": 7.17, "stderr": 3.447, "cost_per_test": 0.001336, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 60.101, "latency": 45.93, "stderr": 3.494, "cost_per_test": 0.006825, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "xiaomi/mimo-v2-flash": { "accuracy": 59.091, "latency": 20.883, "stderr": 3.507, "cost_per_test": 0.001223, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 58.586, "latency": 19.603, "stderr": 3.501, "cost_per_test": 0.002135, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 56.566, "latency": 7.711, "stderr": 3.523, "cost_per_test": 0.011928, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 56.061, "latency": 7.712, "stderr": 3.527, "cost_per_test": 0.004704, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 54.545, "latency": 74.311, "stderr": 3.523, "cost_per_test": 0.001355, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 54.04, "latency": 15.717, "stderr": 3.542, "cost_per_test": 0.011827, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 53.535, "latency": 19.423, "stderr": 3.544, "cost_per_test": 0.002142, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-2-1212": { "accuracy": 52.02, "latency": 20.466, "stderr": 3.55, "cost_per_test": 0.011263, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 51.01, "latency": 4.255, "stderr": 3.438, "cost_per_test": 0.000576, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-2024-05-13": { "accuracy": 50.505, "latency": 10.468, "stderr": 3.553, "cost_per_test": 0.019577, "temperature": 1.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 50.0, "latency": 11.887, "stderr": 3.363, "cost_per_test": 0.012408, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 48.485, "latency": 7.474, "stderr": 3.552, "cost_per_test": 0.002331, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 46.97, "latency": 16.517, "stderr": 3.547, "cost_per_test": 0.008783, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 44.949, "latency": 3.47, "stderr": 3.535, "cost_per_test": 0.000299, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 44.949, "latency": 8.344, "stderr": 3.535, "cost_per_test": 0.000845, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 44.949, "latency": 15.039, "stderr": 3.535, "cost_per_test": 0.00067, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 44.444, "latency": 16.209, "stderr": 3.456, "cost_per_test": null, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 43.939, "latency": 10.003, "stderr": 3.523, "cost_per_test": 0.000366, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 41.414, "latency": 14.853, "stderr": 3.363, "cost_per_test": 0.008222, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 40.909, "latency": 10.759, "stderr": 3.494, "cost_per_test": 0.000746, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 36.364, "latency": 6.978, "stderr": 3.419, "cost_per_test": 0.003004, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 34.848, "latency": 248.163, "stderr": 3.386, "cost_per_test": 0.089386, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 33.333, "latency": 4.791, "stderr": 3.201, "cost_per_test": 0.000652, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "openai/gpt-3.5-turbo": { "accuracy": 28.788, "latency": 2.834, "stderr": 3.201, "cost_per_test": 0.001296, "temperature": 1.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "cohere/command-r-plus": { "accuracy": 28.283, "latency": 8.0, "stderr": 3.183, "cost_per_test": 0.009178, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "poolside/laguna-m.1": { "accuracy": 4.545, "latency": 233.988, "stderr": 1.48, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } }, "diamond_zero_shot_cot": { "openai/gpt-5.6-sol": { "accuracy": 95.455, "latency": 44.206, "stderr": 1.48, "cost_per_test": 0.084288, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 95.455, "latency": 206.939, "stderr": 1.48, "cost_per_test": 0.044383, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 94.949, "latency": 64.024, "stderr": 1.556, "cost_per_test": 0.077857, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 94.949, "latency": 267.316, "stderr": 1.556, "cost_per_test": 0.076987, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 94.444, "latency": 19.256, "stderr": 1.76, "cost_per_test": 0.032582, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 94.444, "latency": 40.592, "stderr": 1.628, "cost_per_test": 0.090786, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 93.939, "latency": 18.517, "stderr": 2.277, "cost_per_test": 0.051126, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 93.939, "latency": 28.557, "stderr": 2.142, "cost_per_test": 0.061459, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 93.939, "latency": 44.083, "stderr": 3.25, "cost_per_test": 0.184706, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 93.434, "latency": 67.496, "stderr": 3.296, "cost_per_test": 0.290028, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 92.929, "latency": 106.252, "stderr": 1.822, "cost_per_test": 0.073175, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 92.929, "latency": 136.373, "stderr": 3.25, "cost_per_test": 0.340421, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 92.929, "latency": 147.589, "stderr": 1.822, "cost_per_test": 0.041673, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 92.929, "latency": 280.541, "stderr": 2.68, "cost_per_test": 0.069677, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 92.929, "latency": 313.788, "stderr": 2.188, "cost_per_test": 0.028001, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 92.424, "latency": 8.36, "stderr": 2.4, "cost_per_test": 0.02089, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 92.424, "latency": 63.916, "stderr": 1.881, "cost_per_test": 0.091397, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 92.424, "latency": 115.375, "stderr": 1.937, "cost_per_test": 0.083665, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 91.919, "latency": 30.09, "stderr": 2.68, "cost_per_test": 0.034726, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 91.919, "latency": 82.463, "stderr": 3.296, "cost_per_test": 0.037824, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 91.919, "latency": 218.997, "stderr": 2.188, "cost_per_test": 0.222481, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 91.414, "latency": 176.161, "stderr": 2.68, "cost_per_test": 0.149003, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 90.909, "latency": 61.485, "stderr": 2.583, "cost_per_test": 0.011853, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 90.909, "latency": 94.785, "stderr": 2.093, "cost_per_test": 0.002959, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 90.909, "latency": 300.77, "stderr": 2.093, "cost_per_test": 0.278848, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 90.404, "latency": 71.088, "stderr": 2.799, "cost_per_test": 0.070514, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 90.404, "latency": 170.513, "stderr": 2.093, "cost_per_test": 0.078189, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 89.899, "latency": 58.717, "stderr": 2.142, "cost_per_test": 0.033188, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 89.899, "latency": 79.801, "stderr": 2.548, "cost_per_test": 0.012785, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.3": { "accuracy": 89.899, "latency": 1082.674, "stderr": 2.142, "cost_per_test": 0.026645, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 89.394, "latency": 82.513, "stderr": 2.188, "cost_per_test": 0.043352, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 89.394, "latency": 201.167, "stderr": 2.319, "cost_per_test": 0.377529, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 88.889, "latency": 129.642, "stderr": 2.233, "cost_per_test": 0.048475, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-0709": { "accuracy": 88.384, "latency": 111.554, "stderr": 2.277, "cost_per_test": 0.143643, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 88.384, "latency": 589.087, "stderr": 3.281, "cost_per_test": 0.084991, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 88.384, "latency": 977.721, "stderr": 2.4, "cost_per_test": 0.138778, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 87.879, "latency": 198.108, "stderr": 2.583, "cost_per_test": 0.015282, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5.2": { "accuracy": 87.879, "latency": 271.144, "stderr": 2.36, "cost_per_test": 0.06444, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 87.374, "latency": 144.698, "stderr": 3.201, "cost_per_test": 0.001659, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3": { "accuracy": 87.374, "latency": 216.036, "stderr": 2.853, "cost_per_test": 0.052834, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 87.374, "latency": 278.169, "stderr": 2.853, "cost_per_test": 0.021457, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 86.869, "latency": 88.114, "stderr": 2.4, "cost_per_test": 0.072084, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 86.869, "latency": 88.527, "stderr": 2.4, "cost_per_test": 0.019122, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 86.869, "latency": 384.478, "stderr": 2.4, "cost_per_test": 0.075911, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 86.364, "latency": 20.422, "stderr": 2.476, "cost_per_test": 0.002352, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 86.364, "latency": 145.221, "stderr": 2.439, "cost_per_test": 0.02817, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 86.364, "latency": 147.729, "stderr": 2.439, "cost_per_test": 0.071283, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 86.364, "latency": 335.213, "stderr": 2.711, "cost_per_test": 0.179663, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 85.859, "latency": 13.756, "stderr": 2.476, "cost_per_test": 0.012083, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 85.859, "latency": 174.88, "stderr": 2.476, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 85.354, "latency": 57.231, "stderr": 2.583, "cost_per_test": 0.033212, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 85.354, "latency": 137.082, "stderr": 3.501, "cost_per_test": 0.691649, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 85.354, "latency": 181.68, "stderr": 3.324, "cost_per_test": 0.035118, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 85.354, "latency": 391.171, "stderr": 3.408, "cost_per_test": 0.418665, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 84.848, "latency": 66.823, "stderr": 2.616, "cost_per_test": 0.002743, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 84.343, "latency": 48.704, "stderr": 2.905, "cost_per_test": 0.003854, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 84.343, "latency": 265.025, "stderr": 3.55, "cost_per_test": 0.006663, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5-thinking": { "accuracy": 83.838, "latency": 250.83, "stderr": 2.616, "cost_per_test": 0.036218, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 83.333, "latency": 82.449, "stderr": 2.649, "cost_per_test": 0.003533, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 83.333, "latency": 101.368, "stderr": 2.905, "cost_per_test": 0.002781, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.1": { "accuracy": 83.249, "latency": 301.653, "stderr": 2.661, "cost_per_test": 0.054832, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 82.828, "latency": 16.842, "stderr": 2.68, "cost_per_test": 0.002266, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 82.323, "latency": 94.328, "stderr": 2.711, "cost_per_test": 0.013029, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.7": { "accuracy": 81.818, "latency": 216.62, "stderr": 2.741, "cost_per_test": 0.025437, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 81.818, "latency": 274.231, "stderr": 2.741, "cost_per_test": 0.003664, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 81.633, "latency": 126.056, "stderr": 3.001, "cost_per_test": 0.119852, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 81.313, "latency": 42.429, "stderr": 2.799, "cost_per_test": 0.011682, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 81.313, "latency": 65.923, "stderr": 2.853, "cost_per_test": 0.006824, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 81.313, "latency": 66.141, "stderr": 3.397, "cost_per_test": 0.013209, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 81.313, "latency": 160.401, "stderr": 2.93, "cost_per_test": 0.013312, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 80.808, "latency": 19.137, "stderr": 2.93, "cost_per_test": 0.002833, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max": { "accuracy": 80.303, "latency": 71.669, "stderr": 2.826, "cost_per_test": 0.015765, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 79.798, "latency": 12.625, "stderr": 3.472, "cost_per_test": 0.056963, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 79.798, "latency": 180.634, "stderr": 3.001, "cost_per_test": 0.018424, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 78.283, "latency": 27.473, "stderr": 3.429, "cost_per_test": 0.001472, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 78.283, "latency": 40.979, "stderr": 2.955, "cost_per_test": 0.003637, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 78.283, "latency": 624.112, "stderr": 3.046, "cost_per_test": 0.023588, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 77.778, "latency": 89.82, "stderr": 2.955, "cost_per_test": 0.012593, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 77.273, "latency": 23.427, "stderr": 3.046, "cost_per_test": 0.002741, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-3": { "accuracy": 76.768, "latency": 33.917, "stderr": 3.046, "cost_per_test": 0.045785, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 76.768, "latency": 108.55, "stderr": 3.046, "cost_per_test": 0.035147, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 76.263, "latency": 13.854, "stderr": 3.046, "cost_per_test": 0.001361, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 76.263, "latency": 52.056, "stderr": 3.024, "cost_per_test": 0.000703, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 75.758, "latency": 34.939, "stderr": 3.108, "cost_per_test": 0.127465, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 75.253, "latency": 31.754, "stderr": 3.067, "cost_per_test": 0.01326, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 75.253, "latency": 124.748, "stderr": 3.108, "cost_per_test": 0.120597, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 75.253, "latency": 170.315, "stderr": 3.067, "cost_per_test": 0.204759, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.6": { "accuracy": 75.253, "latency": 455.539, "stderr": 3.067, "cost_per_test": 0.015448, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 73.737, "latency": 44.979, "stderr": 3.146, "cost_per_test": 0.277401, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 73.737, "latency": 52.454, "stderr": 3.127, "cost_per_test": 0.004527, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-4.5": { "accuracy": 73.737, "latency": 713.892, "stderr": 3.146, "cost_per_test": 0.018777, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 72.727, "latency": 20.631, "stderr": 3.386, "cost_per_test": 0.001675, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 72.727, "latency": 169.828, "stderr": 3.419, "cost_per_test": 0.007436, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 72.222, "latency": 332.583, "stderr": 3.337, "cost_per_test": 0.006971, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 71.212, "latency": 9.589, "stderr": 3.25, "cost_per_test": 0.002112, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 71.212, "latency": 90.985, "stderr": 3.456, "cost_per_test": 0.059337, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 70.707, "latency": 10.602, "stderr": 3.337, "cost_per_test": 0.000831, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 70.707, "latency": 19.822, "stderr": 3.234, "cost_per_test": 0.000425, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 69.697, "latency": 12.541, "stderr": 3.266, "cost_per_test": 0.010833, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 69.697, "latency": 17.986, "stderr": 3.266, "cost_per_test": 0.050314, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 68.687, "latency": 33.342, "stderr": 3.296, "cost_per_test": 0.051403, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 66.667, "latency": 25.835, "stderr": 3.375, "cost_per_test": 0.010086, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 66.162, "latency": 7.377, "stderr": 3.375, "cost_per_test": 0.001135, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 66.162, "latency": 8.579, "stderr": 3.363, "cost_per_test": 0.008588, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 65.657, "latency": 8.956, "stderr": 3.386, "cost_per_test": 0.000392, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 64.646, "latency": 27.347, "stderr": 3.419, "cost_per_test": 0.000969, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 64.141, "latency": 6.63, "stderr": 3.408, "cost_per_test": 0.000469, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 62.626, "latency": 55.277, "stderr": 3.535, "cost_per_test": 0.00502, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 62.121, "latency": 6.771, "stderr": 3.456, "cost_per_test": 0.006229, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 62.121, "latency": 104.648, "stderr": 3.363, "cost_per_test": 0.038764, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 60.606, "latency": 7.706, "stderr": 3.472, "cost_per_test": 0.002606, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 60.101, "latency": 88.877, "stderr": 3.512, "cost_per_test": null, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "xiaomi/mimo-v2-flash": { "accuracy": 59.596, "latency": 20.921, "stderr": 3.553, "cost_per_test": 0.0012, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 57.071, "latency": 2.828, "stderr": 3.531, "cost_per_test": 0.000426, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 56.566, "latency": 7.629, "stderr": 3.523, "cost_per_test": 0.000326, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 56.566, "latency": 46.573, "stderr": 3.539, "cost_per_test": 0.007291, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 55.556, "latency": 23.141, "stderr": 3.539, "cost_per_test": 0.000795, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 55.556, "latency": 64.312, "stderr": 3.544, "cost_per_test": 0.001391, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 53.535, "latency": 14.131, "stderr": 3.55, "cost_per_test": 0.007103, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 53.03, "latency": 13.424, "stderr": 3.55, "cost_per_test": 0.000563, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 51.515, "latency": 7.211, "stderr": 3.552, "cost_per_test": 0.000886, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 51.01, "latency": 16.755, "stderr": 3.523, "cost_per_test": null, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 50.505, "latency": 4.128, "stderr": 3.512, "cost_per_test": 0.0004, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 50.505, "latency": 38.848, "stderr": 3.363, "cost_per_test": 0.026324, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4o-2024-05-13": { "accuracy": 50.0, "latency": 9.688, "stderr": 3.553, "cost_per_test": 0.011302, "temperature": 1.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-2-1212": { "accuracy": 49.495, "latency": 19.89, "stderr": 3.55, "cost_per_test": 0.007736, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 49.495, "latency": 421.926, "stderr": 3.547, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 48.485, "latency": 15.087, "stderr": 3.523, "cost_per_test": 0.004644, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 46.97, "latency": 3.311, "stderr": 3.547, "cost_per_test": 0.000167, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 46.97, "latency": 11.092, "stderr": 3.067, "cost_per_test": 0.00778, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 44.444, "latency": 7.645, "stderr": 3.472, "cost_per_test": 0.000191, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 43.434, "latency": 15.6, "stderr": 3.523, "cost_per_test": 0.000403, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 39.394, "latency": 6.143, "stderr": 3.472, "cost_per_test": 0.001437, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 34.848, "latency": 264.176, "stderr": 3.375, "cost_per_test": 0.101592, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-r-plus": { "accuracy": 33.838, "latency": 8.387, "stderr": 3.266, "cost_per_test": 0.005699, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "openai/gpt-3.5-turbo": { "accuracy": 32.323, "latency": 2.423, "stderr": 3.266, "cost_per_test": 0.000427, "temperature": 1.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 30.808, "latency": 5.115, "stderr": 2.36, "cost_per_test": 0.000322, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 30.808, "latency": 16.089, "stderr": 2.513, "cost_per_test": 0.005158, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } } } }