{ "metadata": { "benchmark": "SAGE", "slug": "sage", "description": "Student Assessment with Generative Evaluation", "benchmark_id": "sage", "family": "sage", "version": "1", "updated": "2026-10-08", "dataset_type": "private", "industry": "education", "tasks": { "overall": "Overall", "PDEs": "Partial Differential Equations", "linear_algebra": "Linear Algebra", "automata": "Automata", "calculus": "Calculus" }, "models": [ "alibaba/qwen3-vl-plus-2025-09-23", "alibaba/qwen3.5-flash", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-27b", "alibaba/qwen3.6-plus", "alibaba/qwen3.7-plus", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-1-20250805-thinking", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4.1-flash", "fireworks/llama4-maverick-instruct-basic", "google/gemini-2.5-flash", "google/gemini-2.5-flash-lite", "google/gemini-2.5-flash-lite-preview-09-2025", "google/gemini-2.5-flash-lite-preview-09-2025-thinking", "google/gemini-2.5-flash-preview-09-2025", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-flash-thinking", "google/gemini-2.5-pro", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "google/gemma-4-31b-it", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-4.7", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "minimax/MiniMax-M3", "mistralai/mistral-large-2512", "mistralai/mistral-medium-3.5", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-nano-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "openai/o3-2025-04-16", "openai/o4-mini-2025-04-16", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "together/meta-llama/Llama-4-Scout-17B-16E-Instruct", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "custom", "mode": "one-shot", "archived": false, "partner": false, "total_models": 91 }, "tasks": { "overall": { "anthropic/claude-opus-4-7": { "accuracy": 56.103, "latency": 124.647, "stderr": 3.368, "cost_per_test": 0.330767, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 55.034, "latency": 93.058, "stderr": 3.411, "cost_per_test": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 54.788, "latency": 185.594, "stderr": 3.347, "cost_per_test": 0.5055, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 53.645, "latency": 63.068, "stderr": 3.385, "cost_per_test": 0.097881, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 52.783, "latency": 47.3, "stderr": 3.414, "cost_per_test": 0.163375, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 52.563, "latency": 75.395, "stderr": 3.416, "cost_per_test": 0.147977, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 52.401, "latency": 83.03, "stderr": 3.467, "cost_per_test": 0.035817, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 52.092, "latency": 87.217, "stderr": 3.403, "cost_per_test": 0.153779, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 51.89, "latency": 116.916, "stderr": 3.4, "cost_per_test": 0.65214, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 51.849, "latency": 42.697, "stderr": 3.404, "cost_per_test": 0.018229, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 51.772, "latency": 113.423, "stderr": 3.412, "cost_per_test": 0.178566, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 51.575, "latency": 160.968, "stderr": 3.339, "cost_per_test": 0.353448, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 51.532, "latency": 76.05, "stderr": 3.954, "cost_per_test": 0.136289, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 51.254, "latency": 132.802, "stderr": 3.415, "cost_per_test": 0.073645, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 50.813, "latency": 141.066, "stderr": 3.403, "cost_per_test": 0.075919, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 50.574, "latency": 118.538, "stderr": 3.436, "cost_per_test": 0.025844, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 50.224, "latency": 270.956, "stderr": 3.429, "cost_per_test": 0.068198, "temperature": null, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 49.885, "latency": 32.592, "stderr": 3.438, "cost_per_test": 0.052847, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 49.865, "latency": 173.179, "stderr": 3.363, "cost_per_test": 0.028736, "temperature": null, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 49.54, "latency": 16.257, "stderr": 3.484, "cost_per_test": 0.007378, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 49.426, "latency": 56.168, "stderr": 3.291, "cost_per_test": 0.192726, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 49.27, "latency": 178.887, "stderr": 3.354, "cost_per_test": 0.115807, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 49.232, "latency": 9.791, "stderr": 3.382, "cost_per_test": 0.026336, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 48.925, "latency": 434.2, "stderr": 3.405, "cost_per_test": 0.588528, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 48.686, "latency": 21.96, "stderr": 3.383, "cost_per_test": 0.045301, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 48.677, "latency": 61.174, "stderr": 3.293, "cost_per_test": 0.068908, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 48.526, "latency": 105.745, "stderr": 3.336, "cost_per_test": 0.61029, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 48.091, "latency": 91.072, "stderr": 3.378, "cost_per_test": 0.006519, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 47.88, "latency": 35.776, "stderr": 3.438, "cost_per_test": 0.01219, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 47.659, "latency": 29.074, "stderr": 3.44, "cost_per_test": 0.032261, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 47.615, "latency": 97.275, "stderr": 3.376, "cost_per_test": 0.012041, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 47.303, "latency": 10.751, "stderr": 3.384, "cost_per_test": 0.010542, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 47.003, "latency": 40.057, "stderr": 3.403, "cost_per_test": 0.037793, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 46.582, "latency": 163.365, "stderr": 3.43, "cost_per_test": 0.234647, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 46.538, "latency": 50.219, "stderr": 3.462, "cost_per_test": 0.061245, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 46.367, "latency": 41.629, "stderr": 3.444, "cost_per_test": 0.282823, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 45.831, "latency": 114.201, "stderr": 3.36, "cost_per_test": 0.332, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 45.783, "latency": 143.279, "stderr": 3.423, "cost_per_test": 0.019327, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 45.58, "latency": 49.948, "stderr": 3.276, "cost_per_test": 0.025502, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 45.577, "latency": 159.25, "stderr": 3.427, "cost_per_test": 0.050269, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 45.05, "latency": 219.909, "stderr": 3.396, "cost_per_test": 0.017193, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 45.002, "latency": 22.727, "stderr": 3.352, "cost_per_test": 0.018815, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 44.86, "latency": 229.267, "stderr": 3.433, "cost_per_test": 0.041429, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-sol": { "accuracy": 44.793, "latency": 31.419, "stderr": 3.364, "cost_per_test": 0.061332, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash": { "accuracy": 44.756, "latency": 40.213, "stderr": 3.418, "cost_per_test": 0.002737, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 44.22, "latency": 51.878, "stderr": 3.331, "cost_per_test": 0.010512, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 44.106, "latency": 31.513, "stderr": 3.432, "cost_per_test": 0.002316, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 43.68, "latency": 58.593, "stderr": 3.349, "cost_per_test": 0.032224, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 43.53, "latency": 136.141, "stderr": 3.429, "cost_per_test": 0.004922, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 43.312, "latency": 152.404, "stderr": 3.115, "cost_per_test": 0.098204, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 43.265, "latency": 52.66, "stderr": 3.387, "cost_per_test": 0.004398, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 43.235, "latency": 98.207, "stderr": 3.16, "cost_per_test": 0.004838, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 42.988, "latency": 36.524, "stderr": 3.338, "cost_per_test": 0.005183, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 42.489, "latency": 85.038, "stderr": 3.46, "cost_per_test": 0.005853, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-vl-plus-2025-09-23": { "accuracy": 42.097, "latency": 74.142, "stderr": 3.384, "cost_per_test": 0.004843, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 41.916, "latency": 52.934, "stderr": 3.411, "cost_per_test": 0.008806, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 41.771, "latency": 48.024, "stderr": 3.315, "cost_per_test": 0.027758, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 41.061, "latency": 27.215, "stderr": 3.395, "cost_per_test": 0.02941, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 40.79, "latency": 290.539, "stderr": 3.348, "cost_per_test": 0.324586, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-thinking": { "accuracy": 39.602, "latency": 39.616, "stderr": 3.422, "cost_per_test": 0.002863, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 39.375, "latency": 27.275, "stderr": 3.339, "cost_per_test": 0.002344, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 39.251, "latency": 177.044, "stderr": 3.384, "cost_per_test": 0.021774, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 38.242, "latency": 66.358, "stderr": 3.425, "cost_per_test": 0.063231, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 38.081, "latency": 34.334, "stderr": 3.087, "cost_per_test": 0.005936, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 37.613, "latency": 55.844, "stderr": 3.356, "cost_per_test": 0.072775, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 36.555, "latency": 198.023, "stderr": 3.216, "cost_per_test": 0.163151, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 36.065, "latency": 210.409, "stderr": 3.209, "cost_per_test": 0.137598, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 35.702, "latency": 43.593, "stderr": 2.933, "cost_per_test": 0.004672, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 35.064, "latency": 28.639, "stderr": 3.364, "cost_per_test": 0.008954, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 35.0, "latency": 17.238, "stderr": 3.274, "cost_per_test": 0.045782, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 34.967, "latency": 169.791, "stderr": 3.316, "cost_per_test": 0.115513, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 34.834, "latency": 32.862, "stderr": 3.029, "cost_per_test": 0.003634, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 32.88, "latency": 16.542, "stderr": 3.141, "cost_per_test": 0.046215, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 32.744, "latency": 206.475, "stderr": 3.258, "cost_per_test": 0.022636, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 32.005, "latency": 104.833, "stderr": 3.215, "cost_per_test": 0.110987, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 31.822, "latency": 66.445, "stderr": 3.134, "cost_per_test": 0.05313, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 31.38, "latency": 17.334, "stderr": 3.024, "cost_per_test": 0.219262, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 31.331, "latency": 77.053, "stderr": 3.02, "cost_per_test": 0.001473, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 30.776, "latency": 32.936, "stderr": 3.124, "cost_per_test": 0.000932, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 30.402, "latency": 312.529, "stderr": 3.147, "cost_per_test": 0.03774, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 30.388, "latency": 40.811, "stderr": 3.188, "cost_per_test": 0.317154, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 30.377, "latency": 38.714, "stderr": 2.834, "cost_per_test": 0.002256, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 29.761, "latency": 31.734, "stderr": 3.043, "cost_per_test": 0.011583, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.6": { "accuracy": 28.9, "latency": 202.474, "stderr": 3.076, "cost_per_test": 0.109757, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 27.973, "latency": 14.674, "stderr": 3.007, "cost_per_test": 0.001132, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-lite": { "accuracy": 27.533, "latency": 13.781, "stderr": 3.124, "cost_per_test": 0.000929, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-0709": { "accuracy": 25.101, "latency": 212.777, "stderr": 2.976, "cost_per_test": 0.405091, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 24.595, "latency": 55.068, "stderr": 2.81, "cost_per_test": 0.00806, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.3": { "accuracy": 19.736, "latency": 96.2, "stderr": 2.647, "cost_per_test": 0.037978, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 15.989, "latency": 6.909, "stderr": 2.43, "cost_per_test": 0.009812, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 11.915, "latency": 5.941, "stderr": 2.165, "cost_per_test": 0.001299, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } }, "PDEs": { "google/gemma-4-31b-it": { "accuracy": 50.794, "latency": 106.921, "stderr": 6.273, "cost_per_test": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 49.206, "latency": 12.879, "stderr": 6.274, "cost_per_test": 0.012636, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 48.413, "latency": 43.479, "stderr": 6.271, "cost_per_test": 0.014128, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 47.619, "latency": 18.889, "stderr": 6.27, "cost_per_test": 0.009314, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 47.619, "latency": 67.069, "stderr": 6.27, "cost_per_test": 0.125985, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 47.619, "latency": 77.028, "stderr": 6.267, "cost_per_test": 0.011059, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 47.619, "latency": 97.811, "stderr": 6.267, "cost_per_test": 0.17995, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 47.619, "latency": 131.155, "stderr": 6.27, "cost_per_test": 0.353409, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 47.619, "latency": 143.824, "stderr": 6.27, "cost_per_test": 0.022988, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 46.032, "latency": 80.626, "stderr": 6.258, "cost_per_test": 0.03527, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 46.032, "latency": 191.629, "stderr": 6.255, "cost_per_test": 0.124815, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 44.444, "latency": 31.274, "stderr": 6.24, "cost_per_test": 0.033761, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 44.444, "latency": 121.425, "stderr": 6.24, "cost_per_test": 0.6238, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 44.444, "latency": 138.901, "stderr": 6.24, "cost_per_test": 0.065649, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 44.444, "latency": 175.937, "stderr": 6.236, "cost_per_test": 0.026757, "temperature": null, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 44.444, "latency": 203.742, "stderr": 6.24, "cost_per_test": 0.494441, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 42.857, "latency": 8.204, "stderr": 6.216, "cost_per_test": 0.021721, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 42.857, "latency": 47.196, "stderr": 6.216, "cost_per_test": 0.057667, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 42.857, "latency": 49.407, "stderr": 6.216, "cost_per_test": 0.142011, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 42.857, "latency": 116.648, "stderr": 6.21, "cost_per_test": 0.003056, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 42.857, "latency": 430.465, "stderr": 6.216, "cost_per_test": 0.567587, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 41.27, "latency": 23.709, "stderr": 6.178, "cost_per_test": 0.020485, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 41.27, "latency": 24.974, "stderr": 6.185, "cost_per_test": 0.050096, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 41.27, "latency": 47.126, "stderr": 6.178, "cost_per_test": 0.020685, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-4-argon": { "accuracy": 41.27, "latency": 61.704, "stderr": 6.185, "cost_per_test": 0.094973, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 41.27, "latency": 64.976, "stderr": 6.185, "cost_per_test": 0.003898, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 41.27, "latency": 66.2, "stderr": 6.185, "cost_per_test": 0.079624, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 41.27, "latency": 124.552, "stderr": 6.185, "cost_per_test": 0.195077, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 41.27, "latency": 188.201, "stderr": 6.185, "cost_per_test": 0.05154, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 41.27, "latency": 196.409, "stderr": 6.185, "cost_per_test": 0.278746, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 41.27, "latency": 200.147, "stderr": 6.178, "cost_per_test": 0.429431, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 39.683, "latency": 21.605, "stderr": 6.139, "cost_per_test": 0.002255, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 39.683, "latency": 42.198, "stderr": 6.148, "cost_per_test": 0.25889, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 39.683, "latency": 47.689, "stderr": 6.148, "cost_per_test": 0.054022, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 39.683, "latency": 52.873, "stderr": 6.148, "cost_per_test": 0.025583, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 39.683, "latency": 54.396, "stderr": 6.148, "cost_per_test": 0.161203, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 39.683, "latency": 63.765, "stderr": 6.148, "cost_per_test": 0.072766, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 39.683, "latency": 71.949, "stderr": 6.139, "cost_per_test": 0.03839, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 39.683, "latency": 91.316, "stderr": 6.148, "cost_per_test": 0.005847, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.5": { "accuracy": 39.683, "latency": 91.895, "stderr": 6.148, "cost_per_test": 0.142524, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 39.683, "latency": 110.565, "stderr": 6.148, "cost_per_test": 0.578727, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 39.683, "latency": 133.32, "stderr": 6.148, "cost_per_test": 0.078775, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 39.683, "latency": 143.966, "stderr": 6.148, "cost_per_test": 0.019522, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 39.683, "latency": 144.753, "stderr": 6.148, "cost_per_test": 0.003876, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 39.683, "latency": 166.337, "stderr": 6.148, "cost_per_test": 0.10951, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 39.683, "latency": 306.764, "stderr": 6.148, "cost_per_test": 0.063769, "temperature": null, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 38.095, "latency": 44.223, "stderr": 6.103, "cost_per_test": 0.042442, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 38.095, "latency": 52.28, "stderr": 6.103, "cost_per_test": 0.010717, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 38.095, "latency": 77.748, "stderr": 6.103, "cost_per_test": 0.005451, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 38.095, "latency": 101.838, "stderr": 6.103, "cost_per_test": 0.28379, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 38.095, "latency": 234.94, "stderr": 6.103, "cost_per_test": 0.041626, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 38.095, "latency": 278.131, "stderr": 6.103, "cost_per_test": 0.016908, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-sol": { "accuracy": 36.508, "latency": 29.991, "stderr": 6.052, "cost_per_test": 0.055244, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 36.508, "latency": 44.522, "stderr": 6.042, "cost_per_test": 0.005441, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 34.921, "latency": 17.422, "stderr": 5.982, "cost_per_test": 0.030408, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 34.921, "latency": 43.826, "stderr": 5.982, "cost_per_test": 0.003103, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 34.921, "latency": 65.101, "stderr": 5.962, "cost_per_test": 0.061403, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 33.333, "latency": 189.296, "stderr": 5.927, "cost_per_test": 0.020906, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.7": { "accuracy": 32.54, "latency": 351.884, "stderr": 5.873, "cost_per_test": 0.434981, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 31.746, "latency": 26.294, "stderr": 5.841, "cost_per_test": 0.023123, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-thinking": { "accuracy": 31.746, "latency": 37.231, "stderr": 5.841, "cost_per_test": 0.002985, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash": { "accuracy": 31.746, "latency": 44.271, "stderr": 5.841, "cost_per_test": 0.00286, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 31.746, "latency": 52.287, "stderr": 5.841, "cost_per_test": 0.284017, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 30.159, "latency": 25.764, "stderr": 5.772, "cost_per_test": 0.006618, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 30.159, "latency": 26.613, "stderr": 5.759, "cost_per_test": 0.002299, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 30.159, "latency": 33.45, "stderr": 5.759, "cost_per_test": 0.001982, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 30.159, "latency": 37.818, "stderr": 5.772, "cost_per_test": 0.007019, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 30.159, "latency": 59.266, "stderr": 5.759, "cost_per_test": 0.00787, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 30.159, "latency": 121.01, "stderr": 5.759, "cost_per_test": 0.001074, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 30.159, "latency": 158.135, "stderr": 5.759, "cost_per_test": 0.167902, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 28.571, "latency": 32.558, "stderr": 5.669, "cost_per_test": 0.002381, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 26.984, "latency": 19.524, "stderr": 5.57, "cost_per_test": 0.032822, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 26.984, "latency": 56.811, "stderr": 5.57, "cost_per_test": 0.02951, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-vl-plus-2025-09-23": { "accuracy": 26.984, "latency": 72.667, "stderr": 5.57, "cost_per_test": 0.003717, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 26.19, "latency": 255.304, "stderr": 5.504, "cost_per_test": 0.024882, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 25.397, "latency": 19.359, "stderr": 5.462, "cost_per_test": 0.147236, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 25.397, "latency": 121.888, "stderr": 5.477, "cost_per_test": 0.153336, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 23.81, "latency": 39.547, "stderr": 5.345, "cost_per_test": 0.008286, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 23.81, "latency": 225.444, "stderr": 5.359, "cost_per_test": 0.171813, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-2.5-flash-lite": { "accuracy": 20.635, "latency": 12.776, "stderr": 5.078, "cost_per_test": 0.001121, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 20.635, "latency": 127.03, "stderr": 5.078, "cost_per_test": 0.115638, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 19.355, "latency": 341.733, "stderr": 5.013, "cost_per_test": 0.040561, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 19.048, "latency": 73.227, "stderr": 4.928, "cost_per_test": 0.052613, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 17.46, "latency": 53.013, "stderr": 4.764, "cost_per_test": 0.006272, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 16.667, "latency": 56.975, "stderr": 4.677, "cost_per_test": 0.000959, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-0709": { "accuracy": 15.079, "latency": 245.263, "stderr": 4.491, "cost_per_test": 0.344283, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 13.492, "latency": 11.273, "stderr": 4.287, "cost_per_test": 0.001268, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 13.492, "latency": 262.598, "stderr": 4.248, "cost_per_test": 0.134364, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 12.698, "latency": 5.622, "stderr": 4.178, "cost_per_test": 0.006058, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.3": { "accuracy": 5.556, "latency": 111.882, "stderr": 2.787, "cost_per_test": 0.043649, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 3.175, "latency": 5.926, "stderr": 2.2, "cost_per_test": 0.000744, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } }, "linear_algebra": { "anthropic/claude-opus-4-8": { "accuracy": 78.253, "latency": 116.985, "stderr": 4.694, "cost_per_test": 0.40882, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 77.039, "latency": 97.423, "stderr": 4.779, "cost_per_test": 0.309413, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 76.056, "latency": 39.301, "stderr": 4.88, "cost_per_test": 0.195615, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 75.361, "latency": 130.677, "stderr": 4.84, "cost_per_test": 0.35649, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 74.436, "latency": 44.698, "stderr": 4.923, "cost_per_test": 0.114649, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 74.205, "latency": 57.323, "stderr": 4.952, "cost_per_test": 0.065497, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-4-argon": { "accuracy": 74.031, "latency": 51.094, "stderr": 4.909, "cost_per_test": 0.093238, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 72.643, "latency": 94.494, "stderr": 5.046, "cost_per_test": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 72.238, "latency": 33.246, "stderr": 5.123, "cost_per_test": 0.172927, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 71.544, "latency": 76.765, "stderr": 5.182, "cost_per_test": 0.55329, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 71.486, "latency": 94.844, "stderr": 5.189, "cost_per_test": 0.055846, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 71.371, "latency": 68.037, "stderr": 5.132, "cost_per_test": 0.524375, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 71.024, "latency": 32.629, "stderr": 5.096, "cost_per_test": 0.014687, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 71.024, "latency": 161.04, "stderr": 5.096, "cost_per_test": 0.031422, "temperature": null, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 70.792, "latency": 76.371, "stderr": 5.201, "cost_per_test": 0.120698, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 70.561, "latency": 70.02, "stderr": 5.119, "cost_per_test": 0.12869, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 70.388, "latency": 43.593, "stderr": 5.146, "cost_per_test": 0.112568, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 69.867, "latency": 18.148, "stderr": 5.138, "cost_per_test": 0.042437, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 69.636, "latency": 100.864, "stderr": 5.222, "cost_per_test": 0.087615, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 69.636, "latency": 127.733, "stderr": 5.165, "cost_per_test": 0.088042, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 69.52, "latency": 9.118, "stderr": 5.164, "cost_per_test": 0.028788, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 68.999, "latency": 194.632, "stderr": 5.228, "cost_per_test": 0.396734, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 68.884, "latency": 38.693, "stderr": 5.179, "cost_per_test": 0.02398, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 68.826, "latency": 101.22, "stderr": 5.226, "cost_per_test": 0.07285, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 68.595, "latency": 91.348, "stderr": 5.213, "cost_per_test": 0.00551, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 68.479, "latency": 60.365, "stderr": 5.224, "cost_per_test": 0.0065, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 68.305, "latency": 22.84, "stderr": 5.215, "cost_per_test": 0.049593, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 68.132, "latency": 221.926, "stderr": 5.254, "cost_per_test": 0.080619, "temperature": null, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 66.859, "latency": 89.83, "stderr": 5.337, "cost_per_test": 0.293876, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 66.802, "latency": 96.704, "stderr": 5.314, "cost_per_test": 0.034489, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 65.934, "latency": 71.823, "stderr": 5.368, "cost_per_test": 0.035292, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 65.067, "latency": 101.578, "stderr": 5.355, "cost_per_test": 0.014122, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 64.372, "latency": 21.402, "stderr": 5.379, "cost_per_test": 0.019286, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 64.141, "latency": 18.915, "stderr": 5.424, "cost_per_test": 0.079054, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 64.025, "latency": 25.326, "stderr": 5.379, "cost_per_test": 0.033758, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 63.852, "latency": 50.49, "stderr": 5.396, "cost_per_test": 0.006765, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 63.736, "latency": 9.53, "stderr": 5.418, "cost_per_test": 0.01033, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 63.216, "latency": 21.032, "stderr": 5.471, "cost_per_test": 0.029377, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 63.216, "latency": 33.283, "stderr": 5.413, "cost_per_test": 0.009948, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 63.158, "latency": 123.106, "stderr": 5.474, "cost_per_test": 0.225951, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 62.927, "latency": 173.849, "stderr": 5.457, "cost_per_test": 0.024337, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 62.695, "latency": 31.536, "stderr": 5.432, "cost_per_test": 0.005581, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 62.348, "latency": 103.446, "stderr": 5.481, "cost_per_test": 0.015403, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 62.29, "latency": 32.298, "stderr": 5.464, "cost_per_test": 0.012613, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 62.117, "latency": 29.309, "stderr": 5.495, "cost_per_test": 0.3535, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 61.596, "latency": 52.517, "stderr": 5.463, "cost_per_test": 0.030379, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 61.423, "latency": 49.184, "stderr": 5.468, "cost_per_test": 0.033352, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 61.249, "latency": 45.238, "stderr": 5.472, "cost_per_test": 0.005317, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 60.671, "latency": 26.941, "stderr": 5.477, "cost_per_test": 0.005836, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 59.977, "latency": 204.949, "stderr": 5.53, "cost_per_test": 0.044938, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 59.746, "latency": 114.854, "stderr": 5.531, "cost_per_test": 0.051599, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 59.456, "latency": 36.385, "stderr": 5.542, "cost_per_test": 0.082922, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 58.936, "latency": 15.659, "stderr": 5.549, "cost_per_test": 0.007622, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 58.878, "latency": 262.039, "stderr": 5.548, "cost_per_test": 0.281116, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 58.762, "latency": 42.202, "stderr": 5.566, "cost_per_test": 0.006857, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-2.5-flash": { "accuracy": 58.473, "latency": 29.009, "stderr": 5.535, "cost_per_test": 0.002669, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 57.316, "latency": 38.478, "stderr": 5.556, "cost_per_test": 0.002481, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 55.986, "latency": 136.886, "stderr": 5.595, "cost_per_test": 0.007778, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 55.813, "latency": 38.342, "stderr": 5.578, "cost_per_test": 0.002429, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 55.813, "latency": 232.901, "stderr": 5.578, "cost_per_test": 0.154143, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 55.697, "latency": 181.336, "stderr": 5.58, "cost_per_test": 0.18401, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 54.135, "latency": 28.93, "stderr": 5.597, "cost_per_test": 0.040058, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-vl-plus-2025-09-23": { "accuracy": 53.673, "latency": 67.761, "stderr": 5.601, "cost_per_test": 0.006561, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 53.441, "latency": 23.095, "stderr": 5.603, "cost_per_test": 0.002453, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 53.268, "latency": 15.449, "stderr": 5.604, "cost_per_test": 0.304378, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 52.863, "latency": 84.486, "stderr": 5.62, "cost_per_test": 0.006619, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 51.938, "latency": 39.772, "stderr": 5.612, "cost_per_test": 0.009215, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 51.88, "latency": 176.535, "stderr": 5.623, "cost_per_test": 0.026418, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 51.533, "latency": 48.364, "stderr": 5.614, "cost_per_test": 0.001538, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 51.417, "latency": 16.36, "stderr": 5.614, "cost_per_test": 0.063363, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 48.641, "latency": 64.809, "stderr": 5.614, "cost_per_test": 0.06133, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 48.409, "latency": 22.788, "stderr": 5.613, "cost_per_test": 0.015989, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.5": { "accuracy": 46.964, "latency": 106.68, "stderr": 5.598, "cost_per_test": 0.108791, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 46.27, "latency": 16.733, "stderr": 5.601, "cost_per_test": 0.062171, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-thinking": { "accuracy": 44.997, "latency": 42.446, "stderr": 5.588, "cost_per_test": 0.002638, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 44.708, "latency": 19.434, "stderr": 5.585, "cost_per_test": 0.001204, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 44.419, "latency": 49.23, "stderr": 5.581, "cost_per_test": 0.009728, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 44.072, "latency": 299.018, "stderr": 5.57, "cost_per_test": 0.04062, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 43.84, "latency": 15.696, "stderr": 5.573, "cost_per_test": 0.001188, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 42.047, "latency": 98.095, "stderr": 5.545, "cost_per_test": 0.127554, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 41.7, "latency": 27.381, "stderr": 5.515, "cost_per_test": 0.011572, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 41.411, "latency": 46.749, "stderr": 5.535, "cost_per_test": 0.072413, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.6": { "accuracy": 41.064, "latency": 180.494, "stderr": 5.487, "cost_per_test": 0.104975, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 39.503, "latency": 176.975, "stderr": 5.416, "cost_per_test": 0.024549, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 39.329, "latency": 49.23, "stderr": 5.459, "cost_per_test": 0.083369, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 37.594, "latency": 44.045, "stderr": 5.441, "cost_per_test": 0.416739, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-0709": { "accuracy": 35.223, "latency": 172.123, "stderr": 5.365, "cost_per_test": 0.503586, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 28.918, "latency": 6.518, "stderr": 5.093, "cost_per_test": 0.014611, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite": { "accuracy": 28.629, "latency": 18.12, "stderr": 5.077, "cost_per_test": 0.000791, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 21.457, "latency": 6.047, "stderr": 4.611, "cost_per_test": 0.001468, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.3": { "accuracy": 18.334, "latency": 103.302, "stderr": 4.105, "cost_per_test": 0.036954, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } }, "automata": { "google/gemini-4-argon": { "accuracy": 45.635, "latency": 76.406, "stderr": 6.386, "cost_per_test": 0.10543, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-vl-plus-2025-09-23": { "accuracy": 45.635, "latency": 81.998, "stderr": 6.376, "cost_per_test": 0.004252, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 45.238, "latency": 96.64, "stderr": 6.342, "cost_per_test": 0.036891, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash": { "accuracy": 44.048, "latency": 47.36, "stderr": 6.355, "cost_per_test": 0.002682, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 43.651, "latency": 59.764, "stderr": 6.33, "cost_per_test": 0.009334, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 43.65, "latency": 145.362, "stderr": 6.318, "cost_per_test": 0.329478, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 43.254, "latency": 48.336, "stderr": 6.336, "cost_per_test": 0.019315, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 43.254, "latency": 59.246, "stderr": 6.328, "cost_per_test": 0.175188, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 43.254, "latency": 139.345, "stderr": 6.281, "cost_per_test": 0.219924, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 42.857, "latency": 284.178, "stderr": 6.36, "cost_per_test": 0.060206, "temperature": null, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 42.064, "latency": 14.224, "stderr": 6.255, "cost_per_test": 0.005199, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-thinking": { "accuracy": 42.064, "latency": 39.172, "stderr": 6.329, "cost_per_test": 0.002966, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 41.666, "latency": 77.76, "stderr": 6.317, "cost_per_test": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 41.666, "latency": 236.054, "stderr": 6.311, "cost_per_test": 0.61324, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 41.27, "latency": 195.033, "stderr": 6.284, "cost_per_test": 0.093136, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 40.476, "latency": 66.582, "stderr": 6.273, "cost_per_test": 0.046792, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 40.476, "latency": 84.742, "stderr": 6.345, "cost_per_test": 0.044921, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 40.476, "latency": 91.557, "stderr": 8.871, "cost_per_test": 0.151694, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 39.682, "latency": 115.524, "stderr": 6.262, "cost_per_test": 0.205378, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 39.682, "latency": 152.558, "stderr": 6.183, "cost_per_test": 0.779331, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 39.682, "latency": 158.642, "stderr": 6.227, "cost_per_test": 0.06767, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 38.889, "latency": 50.623, "stderr": 6.166, "cost_per_test": 0.037181, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 38.492, "latency": 27.741, "stderr": 6.367, "cost_per_test": 0.051282, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 38.096, "latency": 93.82, "stderr": 6.223, "cost_per_test": 0.152698, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 38.095, "latency": 152.08, "stderr": 6.227, "cost_per_test": 0.274423, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 37.698, "latency": 135.104, "stderr": 6.177, "cost_per_test": 0.007607, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 37.302, "latency": 26.42, "stderr": 6.187, "cost_per_test": 0.025048, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 37.302, "latency": 53.38, "stderr": 6.228, "cost_per_test": 0.23608, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 37.302, "latency": 115.085, "stderr": 6.222, "cost_per_test": 0.020054, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 36.905, "latency": 38.076, "stderr": 6.162, "cost_per_test": 0.020412, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 36.508, "latency": 247.912, "stderr": 6.182, "cost_per_test": 0.037724, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 35.714, "latency": 174.695, "stderr": 6.073, "cost_per_test": 0.047668, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 35.318, "latency": 12.052, "stderr": 6.137, "cost_per_test": 0.028498, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 35.318, "latency": 34.456, "stderr": 6.118, "cost_per_test": 0.002213, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 35.318, "latency": 34.916, "stderr": 6.137, "cost_per_test": 0.033644, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.3": { "accuracy": 35.318, "latency": 73.416, "stderr": 6.199, "cost_per_test": 0.033332, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 35.318, "latency": 170.579, "stderr": 6.137, "cost_per_test": 0.199244, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 35.318, "latency": 182.426, "stderr": 6.131, "cost_per_test": 0.023056, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 34.921, "latency": 79.312, "stderr": 6.195, "cost_per_test": 0.005093, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 34.921, "latency": 126.784, "stderr": 6.061, "cost_per_test": 0.003112, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 34.92, "latency": 22.757, "stderr": 6.195, "cost_per_test": 0.04337, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 34.92, "latency": 677.504, "stderr": 6.195, "cost_per_test": 0.801263, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 34.524, "latency": 32.116, "stderr": 5.983, "cost_per_test": 0.00228, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 34.524, "latency": 138.632, "stderr": 6.002, "cost_per_test": 0.727769, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 34.127, "latency": 182.559, "stderr": 6.078, "cost_per_test": 0.02803, "temperature": null, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 34.127, "latency": 207.746, "stderr": 6.062, "cost_per_test": 0.010334, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-sol": { "accuracy": 33.73, "latency": 45.352, "stderr": 5.985, "cost_per_test": 0.049697, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite": { "accuracy": 33.334, "latency": 10.448, "stderr": 6.023, "cost_per_test": 0.000876, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 33.334, "latency": 32.772, "stderr": 6.173, "cost_per_test": 0.008672, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 33.334, "latency": 89.374, "stderr": 6.041, "cost_per_test": 0.089769, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 32.936, "latency": 31.551, "stderr": 6.098, "cost_per_test": 0.00983, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 32.54, "latency": 74.808, "stderr": 5.991, "cost_per_test": 0.221359, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 32.54, "latency": 150.936, "stderr": 5.991, "cost_per_test": 0.418335, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 32.54, "latency": 165.302, "stderr": 6.027, "cost_per_test": 0.017998, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 32.54, "latency": 187.145, "stderr": 5.991, "cost_per_test": 0.018476, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.5": { "accuracy": 32.54, "latency": 280.804, "stderr": 6.132, "cost_per_test": 0.084413, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 32.143, "latency": 62.434, "stderr": 5.937, "cost_per_test": 0.068462, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 32.143, "latency": 164.331, "stderr": 6.081, "cost_per_test": 0.089931, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 32.143, "latency": 217.3, "stderr": 5.952, "cost_per_test": 0.134564, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 31.35, "latency": 70.071, "stderr": 5.772, "cost_per_test": 0.010871, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 30.952, "latency": 22.4, "stderr": 5.896, "cost_per_test": 0.000634, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 30.952, "latency": 257.694, "stderr": 5.968, "cost_per_test": 0.257662, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 30.159, "latency": 54.582, "stderr": 5.701, "cost_per_test": 0.066288, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 30.159, "latency": 187.29, "stderr": 5.766, "cost_per_test": 0.13363, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 30.158, "latency": 113.22, "stderr": 5.884, "cost_per_test": 0.010942, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 29.762, "latency": 33.513, "stderr": 5.852, "cost_per_test": 0.004528, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 29.762, "latency": 50.802, "stderr": 5.834, "cost_per_test": 0.002438, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 29.762, "latency": 51.314, "stderr": 5.782, "cost_per_test": 0.027903, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 29.365, "latency": 23.07, "stderr": 5.832, "cost_per_test": 0.016674, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 28.968, "latency": 9.844, "stderr": 5.86, "cost_per_test": 0.008661, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 28.174, "latency": 58.277, "stderr": 5.654, "cost_per_test": 0.026942, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 27.778, "latency": 61.3, "stderr": 5.711, "cost_per_test": 0.045446, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 27.778, "latency": 296.836, "stderr": 5.743, "cost_per_test": 0.032038, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 26.588, "latency": 17.054, "stderr": 5.653, "cost_per_test": 0.00094, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-0709": { "accuracy": 25.0, "latency": 220.944, "stderr": 5.546, "cost_per_test": 0.367405, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 23.81, "latency": 17.56, "stderr": 5.414, "cost_per_test": 0.044766, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 23.412, "latency": 38.242, "stderr": 4.736, "cost_per_test": 0.004953, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 22.222, "latency": 240.19, "stderr": 5.327, "cost_per_test": 0.09075, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 21.825, "latency": 26.102, "stderr": 5.266, "cost_per_test": 0.250705, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 21.428, "latency": 189.656, "stderr": 4.715, "cost_per_test": 0.112253, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 20.238, "latency": 13.741, "stderr": 5.121, "cost_per_test": 0.04246, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 18.254, "latency": 86.624, "stderr": 4.915, "cost_per_test": 0.005948, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 17.064, "latency": 32.868, "stderr": 4.821, "cost_per_test": 0.010474, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 15.476, "latency": 17.193, "stderr": 4.59, "cost_per_test": 0.206172, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 14.682, "latency": 20.79, "stderr": 4.524, "cost_per_test": 0.003204, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 12.302, "latency": 61.786, "stderr": 4.174, "cost_per_test": 0.001806, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 11.905, "latency": 62.961, "stderr": 4.152, "cost_per_test": 0.00818, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 11.112, "latency": 5.85, "stderr": 4.009, "cost_per_test": 0.001684, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 8.333, "latency": 36.462, "stderr": 3.535, "cost_per_test": 0.004148, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 6.35, "latency": 8.586, "stderr": 3.125, "cost_per_test": 0.008768, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 5.159, "latency": 44.351, "stderr": 2.825, "cost_per_test": 0.002358, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null } }, "calculus": { "google/gemini-4-argon": { "accuracy": 84.877, "latency": 34.392, "stderr": 4.364, "cost_per_test": 0.054821, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 83.642, "latency": 8.815, "stderr": 4.441, "cost_per_test": 0.039773, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 83.642, "latency": 70.315, "stderr": 4.491, "cost_per_test": 0.479685, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 83.642, "latency": 87.434, "stderr": 4.491, "cost_per_test": 1.173994, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 82.407, "latency": 91.759, "stderr": 4.603, "cost_per_test": 0.335486, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 79.938, "latency": 23.443, "stderr": 4.783, "cost_per_test": 0.683311, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 79.938, "latency": 25.764, "stderr": 4.676, "cost_per_test": 0.165991, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 79.938, "latency": 45.261, "stderr": 4.772, "cost_per_test": 0.323515, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 79.63, "latency": 18.566, "stderr": 5.106, "cost_per_test": 0.020704, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 78.395, "latency": 57.524, "stderr": 5.215, "cost_per_test": 0.070287, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 78.395, "latency": 86.271, "stderr": 5.28, "cost_per_test": 0.292098, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 77.16, "latency": 58.451, "stderr": 5.312, "cost_per_test": 0.524527, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 75.926, "latency": 121.638, "stderr": 5.396, "cost_per_test": 0.367952, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 75.309, "latency": 15.152, "stderr": 5.764, "cost_per_test": 0.047167, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 75.0, "latency": 27.696, "stderr": 4.991, "cost_per_test": 0.176051, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 74.691, "latency": 56.224, "stderr": 5.469, "cost_per_test": 0.05685, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 73.457, "latency": 103.527, "stderr": 5.53, "cost_per_test": 0.145267, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 72.531, "latency": 263.947, "stderr": 5.081, "cost_per_test": 0.822692, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 72.222, "latency": 54.876, "stderr": 5.571, "cost_per_test": 0.112212, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 71.914, "latency": 110.79, "stderr": 5.866, "cost_per_test": 0.12241, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 71.605, "latency": 92.023, "stderr": 6.022, "cost_per_test": 0.019685, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 70.988, "latency": 60.109, "stderr": 5.5, "cost_per_test": 0.014869, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 70.988, "latency": 150.536, "stderr": 5.601, "cost_per_test": 0.105882, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 70.679, "latency": 26.461, "stderr": 5.904, "cost_per_test": 0.051615, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 70.679, "latency": 62.783, "stderr": 5.808, "cost_per_test": 0.019416, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 70.371, "latency": 256.707, "stderr": 6.069, "cost_per_test": 0.288525, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 69.753, "latency": 119.216, "stderr": 5.652, "cost_per_test": 0.014167, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 69.444, "latency": 122.682, "stderr": 5.961, "cost_per_test": 0.036851, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 67.284, "latency": 46.739, "stderr": 5.681, "cost_per_test": 0.157303, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 64.506, "latency": 47.895, "stderr": 6.102, "cost_per_test": 0.019177, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 62.963, "latency": 94.04, "stderr": 6.225, "cost_per_test": 0.060657, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 62.037, "latency": 174.449, "stderr": 6.102, "cost_per_test": 0.137909, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 61.729, "latency": 144.533, "stderr": 6.324, "cost_per_test": 0.193613, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 60.802, "latency": 80.117, "stderr": 6.13, "cost_per_test": 0.249092, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 60.494, "latency": 9.923, "stderr": 6.377, "cost_per_test": 0.012024, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 60.494, "latency": 207.52, "stderr": 6.324, "cost_per_test": 0.458947, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 59.568, "latency": 696.783, "stderr": 6.008, "cost_per_test": 0.101612, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 59.259, "latency": 31.529, "stderr": 6.315, "cost_per_test": 0.055115, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemma-4-31b-it": { "accuracy": 59.259, "latency": 88.767, "stderr": 6.315, "cost_per_test": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 59.259, "latency": 134.807, "stderr": 6.36, "cost_per_test": 0.082637, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 58.333, "latency": 19.833, "stderr": 6.102, "cost_per_test": 0.002188, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 58.333, "latency": 39.018, "stderr": 6.139, "cost_per_test": 0.024556, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash": { "accuracy": 55.864, "latency": 35.521, "stderr": 6.148, "cost_per_test": 0.005276, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 55.556, "latency": 49.598, "stderr": 6.395, "cost_per_test": 0.005121, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 55.556, "latency": 145.43, "stderr": 6.306, "cost_per_test": 0.06546, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 54.012, "latency": 65.915, "stderr": 6.484, "cost_per_test": 0.005383, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-thinking": { "accuracy": 51.852, "latency": 41.551, "stderr": 6.386, "cost_per_test": 0.005404, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 49.383, "latency": 62.372, "stderr": 6.324, "cost_per_test": 0.097577, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 48.148, "latency": 65.817, "stderr": 6.333, "cost_per_test": 0.058546, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 46.605, "latency": 75.8, "stderr": 6.475, "cost_per_test": 0.235685, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 46.605, "latency": 147.142, "stderr": 6.51, "cost_per_test": 0.056504, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 46.296, "latency": 85.079, "stderr": 6.516, "cost_per_test": 1.424207, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 44.444, "latency": 138.842, "stderr": 6.225, "cost_per_test": 0.011357, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-luna": { "accuracy": 44.136, "latency": 96.286, "stderr": 6.449, "cost_per_test": 0.013009, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 40.123, "latency": 119.528, "stderr": 6.472, "cost_per_test": 0.130055, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 39.815, "latency": 40.17, "stderr": 6.484, "cost_per_test": 0.010142, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 37.654, "latency": 23.802, "stderr": 6.393, "cost_per_test": 0.199573, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 36.42, "latency": 45.785, "stderr": 6.34, "cost_per_test": 0.001696, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 33.333, "latency": 20.801, "stderr": 6.225, "cost_per_test": 0.011459, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 33.333, "latency": 72.463, "stderr": 6.225, "cost_per_test": 0.10162, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 33.333, "latency": 81.528, "stderr": 6.225, "cost_per_test": 0.011822, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-0709": { "accuracy": 29.938, "latency": 169.15, "stderr": 6.125, "cost_per_test": 1.176585, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 29.321, "latency": 24.759, "stderr": 5.923, "cost_per_test": 0.196354, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 29.012, "latency": 59.864, "stderr": 5.954, "cost_per_test": 0.042124, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 28.086, "latency": 30.662, "stderr": 5.876, "cost_per_test": 0.0019, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 27.16, "latency": 21.745, "stderr": 5.909, "cost_per_test": 0.022455, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 26.852, "latency": 25.246, "stderr": 5.818, "cost_per_test": 0.984331, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 24.691, "latency": 42.558, "stderr": 5.822, "cost_per_test": 0.043415, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 22.839, "latency": 143.574, "stderr": 5.48, "cost_per_test": 0.129492, "temperature": null, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 20.988, "latency": 45.07, "stderr": 4.183, "cost_per_test": 0.002745, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 19.136, "latency": 18.058, "stderr": 5.117, "cost_per_test": 0.010491, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 18.519, "latency": 125.145, "stderr": 5.315, "cost_per_test": 0.072804, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 17.284, "latency": 210.765, "stderr": 5.24, "cost_per_test": 0.055617, "temperature": null, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 15.741, "latency": 115.804, "stderr": 5.049, "cost_per_test": 0.020731, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3-vl-plus-2025-09-23": { "accuracy": 15.432, "latency": 96.375, "stderr": 4.81, "cost_per_test": 0.01699, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-lite": { "accuracy": 13.889, "latency": 13.337, "stderr": 4.298, "cost_per_test": 0.001699, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 13.272, "latency": 345.421, "stderr": 4.785, "cost_per_test": 0.051045, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 12.963, "latency": 129.213, "stderr": 4.581, "cost_per_test": 0.214304, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 8.025, "latency": 125.364, "stderr": 3.974, "cost_per_test": 0.053686, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 7.716, "latency": 72.05, "stderr": 3.55, "cost_per_test": 0.016007, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 7.716, "latency": 78.88, "stderr": 3.55, "cost_per_test": 0.004291, "temperature": null, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 5.247, "latency": 68.888, "stderr": 3.146, "cost_per_test": 0.014759, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 2.778, "latency": 73.75, "stderr": 2.66, "cost_per_test": 0.073682, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 2.778, "latency": 75.714, "stderr": 2.66, "cost_per_test": 0.018248, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 0.0, "latency": 101.608, "stderr": 0.0, "cost_per_test": 0.010832, "temperature": 1.0, "top_p": null, "max_output_tokens": 30000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null } } } }