{ "metadata": { "benchmark": "Vals Index", "slug": "vals_index_v1_2", "description": "Benchmark consisting of a weighted performance across finance and coding tasks. Showing the potential impact that LLM's can have on the economy.", "benchmark_id": "vals_index_v1_2", "family": "vals_index", "version": "1.2", "updated": "2026-08-11", "dataset_type": "private", "industry": "index", "tasks": { "overall": "Overall", "finance_agent": "Finance Agent v2", "corp_fin_v2": "CorpFin v2", "swebench": "SWE-bench", "terminal_bench_2_1": "Terminal-Bench 2.1", "vibe_code_bench": "Vibe Code Bench" }, "models": [ "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.7-plus", "alibaba/qwen3.8-max", "ant/ling-3.0-flash-2607", "anthropic/claude-fable-5", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/mistral-medium-3.5", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "poolside/laguna-m.1", "poolside/laguna-xs.2", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "zai/glm-5.1", "zai/glm-5.2" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "platform", "mode": "agentic", "archived": true, "partner": false, "total_models": 45 }, "tasks": { "overall": { "anthropic/claude-fable-5": { "accuracy": 75.145, "latency": 1050.682, "stderr": 0.637, "cost_per_test": 11.002463, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 74.82, "latency": 1182.436, "stderr": 1.353, "cost_per_test": 8.538371, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 74.7, "latency": 1223.498, "stderr": 0.96, "cost_per_test": 2.34403, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 73.118, "latency": 753.48, "stderr": 0.743, "cost_per_test": 7.457146, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 71.877, "latency": 609.631, "stderr": 1.116, "cost_per_test": 0.696987, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 70.358, "latency": 1330.504, "stderr": 0.902, "cost_per_test": 7.522317, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 69.878, "latency": 554.38, "stderr": 0.836, "cost_per_test": 0.222066, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 68.608, "latency": 1324.286, "stderr": 0.996, "cost_per_test": 9.012369, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 68.405, "latency": 388.518, "stderr": 1.272, "cost_per_test": 0.498787, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.5": { "accuracy": 67.951, "latency": 684.387, "stderr": 1.536, "cost_per_test": 4.60358, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 66.099, "latency": 761.527, "stderr": 1.268, "cost_per_test": 6.135313, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 65.469, "latency": 2758.566, "stderr": 1.625, "cost_per_test": 2.676811, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 65.299, "latency": 388.451, "stderr": 1.406, "cost_per_test": 1.248659, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 65.135, "latency": 352.049, "stderr": 1.715, "cost_per_test": 1.109172, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 65.017, "latency": 1122.395, "stderr": 1.422, "cost_per_test": 2.076891, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 64.915, "latency": 743.677, "stderr": 1.502, "cost_per_test": 1.420371, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 63.952, "latency": 859.572, "stderr": 1.308, "cost_per_test": 0.061876, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 62.748, "latency": 364.275, "stderr": 1.606, "cost_per_test": 1.401628, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 60.065, "latency": 711.832, "stderr": 1.525, "cost_per_test": 2.15182, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 58.939, "latency": 1477.674, "stderr": 1.817, "cost_per_test": 1.500191, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 57.487, "latency": 1831.709, "stderr": 1.572, "cost_per_test": 5.223348, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 55.615, "latency": 1326.907, "stderr": 1.685, "cost_per_test": 0.828183, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 55.166, "latency": 1189.192, "stderr": 1.924, "cost_per_test": 0.705307, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 53.77, "latency": 435.112, "stderr": 1.591, "cost_per_test": 1.350398, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 52.452, "latency": 898.864, "stderr": 1.607, "cost_per_test": 0.863172, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 52.425, "latency": 1050.167, "stderr": 2.054, "cost_per_test": 0.65965, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 52.331, "latency": 832.609, "stderr": 1.628, "cost_per_test": 0.351425, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 51.57, "latency": 585.32, "stderr": 1.569, "cost_per_test": 0.039823, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 50.997, "latency": 342.247, "stderr": 1.623, "cost_per_test": 0.416812, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 50.744, "latency": 839.412, "stderr": 1.626, "cost_per_test": 0.091426, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 50.374, "latency": 928.928, "stderr": 1.472, "cost_per_test": 0.188172, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 49.546, "latency": 353.286, "stderr": 1.545, "cost_per_test": 0.397051, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling": { "accuracy": 49.284, "latency": 1247.29, "stderr": 1.292, "cost_per_test": 0.84133, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 48.887, "latency": 838.547, "stderr": 1.455, "cost_per_test": 1.552897, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 46.635, "latency": 929.692, "stderr": 1.866, "cost_per_test": 0.324322, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 46.48, "latency": 366.757, "stderr": 1.364, "cost_per_test": 0.68602, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 43.987, "latency": 638.492, "stderr": 1.263, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 41.56, "latency": 782.736, "stderr": 1.178, "cost_per_test": 0.720828, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 40.903, "latency": 376.932, "stderr": 1.023, "cost_per_test": 0.558143, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 39.496, "latency": 223.886, "stderr": 0.772, "cost_per_test": 0.540067, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 38.02, "latency": 402.172, "stderr": 0.747, "cost_per_test": 0.096931, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 37.814, "latency": 1003.73, "stderr": 1.057, "cost_per_test": 4.815687, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 36.2, "latency": 209.083, "stderr": 0.741, "cost_per_test": 0.17062, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 33.334, "latency": 1212.296, "stderr": 0.934, "cost_per_test": 0.789689, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 30.041, "latency": 747.181, "stderr": 1.009, "cost_per_test": 0.316248, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } }, "vibe_code_bench": { "anthropic/claude-fable-5": { "accuracy": 93.467, "latency": 3711.026, "stderr": 1.473, "cost_per_test": 41.737416, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 91.271, "latency": 4041.504, "stderr": 2.21, "cost_per_test": 9.904582, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 89.869, "latency": 4549.256, "stderr": 2.244, "cost_per_test": 28.380692, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 89.34, "latency": 2033.047, "stderr": 2.575, "cost_per_test": 33.403634, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 86.902, "latency": 4001.424, "stderr": 3.468, "cost_per_test": 38.07813, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 85.664, "latency": 3900.597, "stderr": 6.078, "cost_per_test": 34.089223, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 82.854, "latency": 1195.221, "stderr": 4.137, "cost_per_test": 1.529642, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 81.053, "latency": 1551.831, "stderr": 3.009, "cost_per_test": 0.748037, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 77.701, "latency": 2141.902, "stderr": 5.336, "cost_per_test": 21.407199, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 77.099, "latency": 1911.567, "stderr": 7.028, "cost_per_test": 16.661911, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 74.911, "latency": 10135.139, "stderr": 6.9, "cost_per_test": 10.030932, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 74.63, "latency": 2907.14, "stderr": 5.324, "cost_per_test": 0.189928, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 72.881, "latency": 3748.167, "stderr": 6.217, "cost_per_test": 8.488514, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 72.818, "latency": 959.495, "stderr": 7.59, "cost_per_test": 4.596845, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 72.754, "latency": 830.057, "stderr": 5.807, "cost_per_test": 4.102194, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 72.412, "latency": 615.428, "stderr": 5.168, "cost_per_test": 0.923283, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 69.941, "latency": 1523.201, "stderr": 6.273, "cost_per_test": 3.041993, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 56.54, "latency": 5073.23, "stderr": 7.963, "cost_per_test": 6.453899, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 56.223, "latency": 1572.118, "stderr": 6.546, "cost_per_test": 5.907892, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 55.905, "latency": 3418.92, "stderr": 7.421, "cost_per_test": 2.205991, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 54.709, "latency": 893.074, "stderr": 7.002, "cost_per_test": 2.540385, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 52.88, "latency": 6683.491, "stderr": 6.854, "cost_per_test": 20.253775, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 48.148, "latency": 1580.348, "stderr": 6.086, "cost_per_test": 0.068395, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 46.94, "latency": 2231.75, "stderr": 7.011, "cost_per_test": 1.082348, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 42.523, "latency": 2055.117, "stderr": 8.901, "cost_per_test": 1.191299, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 42.323, "latency": 2967.768, "stderr": 8.392, "cost_per_test": 1.925351, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 36.321, "latency": 2403.764, "stderr": 6.374, "cost_per_test": 0.166747, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 34.214, "latency": 1211.906, "stderr": 6.444, "cost_per_test": 3.825202, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 32.804, "latency": 2014.732, "stderr": 7.069, "cost_per_test": 2.894911, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 32.772, "latency": 512.273, "stderr": 6.978, "cost_per_test": 1.006326, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 30.032, "latency": 3276.224, "stderr": 8.282, "cost_per_test": 1.277298, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 28.253, "latency": 2289.977, "stderr": 6.126, "cost_per_test": 5.447636, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 21.954, "latency": 806.66, "stderr": 6.673, "cost_per_test": 0.942067, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 19.039, "latency": 2949.469, "stderr": 5.403, "cost_per_test": 0.432264, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.3": { "accuracy": 15.475, "latency": 589.407, "stderr": 5.576, "cost_per_test": 1.28472, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 13.277, "latency": 2891.761, "stderr": 4.646, "cost_per_test": 1.728665, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 11.719, "latency": 1377.326, "stderr": 4.172, "cost_per_test": 2.819109, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 9.882, "latency": 775.882, "stderr": 3.549, "cost_per_test": 1.306474, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 7.104, "latency": 2024.461, "stderr": 3.207, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 5.465, "latency": 1500.542, "stderr": 2.058, "cost_per_test": 0.922398, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-m.1": { "accuracy": 5.264, "latency": 1997.076, "stderr": 2.352, "cost_per_test": 2.228448, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 1.515, "latency": 301.651, "stderr": 1.515, "cost_per_test": 0.771885, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 1.24, "latency": 2470.533, "stderr": 1.24, "cost_per_test": 16.735503, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 572.711, "stderr": 0.0, "cost_per_test": 0.51, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 1005.598, "stderr": 0.0, "cost_per_test": 0.330448, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null } }, "swebench": { "openai/gpt-5.6-sol": { "accuracy": 99.02, "latency": 237.142, "stderr": 0.976, "cost_per_test": 1.311279, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "anthropic/claude-fable-5": { "accuracy": 97.059, "latency": 441.198, "stderr": 1.673, "cost_per_test": 2.046736, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "kimi/kimi-k3": { "accuracy": 95.098, "latency": 611.386, "stderr": 2.138, "cost_per_test": 0.879924, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "anthropic/claude-opus-5": { "accuracy": 95.098, "latency": 665.89, "stderr": 2.138, "cost_per_test": 1.554764, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 93.137, "latency": 184.771, "stderr": 2.503, "cost_per_test": 0.051004, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 89.216, "latency": 630.305, "stderr": 3.071, "cost_per_test": 1.923224, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 88.235, "latency": 233.186, "stderr": 3.19, "cost_per_test": 0.011882, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 87.255, "latency": 1553.607, "stderr": 3.302, "cost_per_test": 1.577912, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "grok/grok-4.5": { "accuracy": 86.275, "latency": 210.846, "stderr": 3.407, "cost_per_test": 0.606056, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "meta/muse_spark_1_2": { "accuracy": 85.294, "latency": 638.55, "stderr": 3.507, "cost_per_test": 0.644011, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "zai/glm-5.2": { "accuracy": 83.333, "latency": 650.517, "stderr": 3.69, "cost_per_test": 0.710002, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openai/gpt-5.5": { "accuracy": 82.6, "latency": 426.425, "stderr": 1.695, "cost_per_test": 1.361765, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 82.353, "latency": 508.652, "stderr": 3.775, "cost_per_test": 2.999546, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "meta/muse_spark_1_1": { "accuracy": 78.431, "latency": 238.11, "stderr": 4.072, "cost_per_test": 0.352022, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 77.451, "latency": 633.682, "stderr": 4.138, "cost_per_test": 1.706873, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "google/gemini-3.6-flash": { "accuracy": 77.451, "latency": 1340.979, "stderr": 4.138, "cost_per_test": 1.380993, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "google/gemini-3.5-flash": { "accuracy": 75.49, "latency": 230.39, "stderr": 4.259, "cost_per_test": 0.953882, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 75.49, "latency": 305.548, "stderr": 4.259, "cost_per_test": 0.13413, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 75.49, "latency": 574.117, "stderr": 4.259, "cost_per_test": 0.436763, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 75.49, "latency": 1097.115, "stderr": 4.259, "cost_per_test": 1.749654, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "thinkingmachines/inkling": { "accuracy": 75.49, "latency": 1420.277, "stderr": 4.259, "cost_per_test": 0.585966, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "kimi/kimi-k2.6": { "accuracy": 74.51, "latency": 396.661, "stderr": 4.315, "cost_per_test": 0.322908, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 73.529, "latency": 142.056, "stderr": 4.368, "cost_per_test": 0.286403, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 72.549, "latency": 329.202, "stderr": 4.419, "cost_per_test": 0.820116, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "zai/glm-5.1": { "accuracy": 72.549, "latency": 598.277, "stderr": 4.419, "cost_per_test": 0.534539, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "grok/grok-4.3": { "accuracy": 70.588, "latency": 126.579, "stderr": 4.512, "cost_per_test": 0.23972, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 70.588, "latency": 487.435, "stderr": 4.512, "cost_per_test": 0.901327, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "minimax/MiniMax-M3": { "accuracy": 69.608, "latency": 765.232, "stderr": 4.554, "cost_per_test": 0.508278, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 69.608, "latency": 961.544, "stderr": 4.554, "cost_per_test": 0.525961, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 68.627, "latency": 327.172, "stderr": 4.594, "cost_per_test": 0.306186, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 68.627, "latency": 577.657, "stderr": 4.594, "cost_per_test": 0.339998, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 67.647, "latency": 281.133, "stderr": 4.632, "cost_per_test": 0.107316, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 67.647, "latency": 328.551, "stderr": 4.632, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 67.647, "latency": 355.599, "stderr": 4.632, "cost_per_test": 0.589598, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 66.667, "latency": 151.343, "stderr": 4.668, "cost_per_test": 0.586871, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 66.667, "latency": 518.711, "stderr": 4.668, "cost_per_test": 0.026452, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 66.667, "latency": 602.556, "stderr": 4.668, "cost_per_test": 0.207597, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 66.667, "latency": 1037.61, "stderr": 4.668, "cost_per_test": 3.384593, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 64.706, "latency": 264.186, "stderr": 4.732, "cost_per_test": 0.407017, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 64.706, "latency": 284.365, "stderr": 4.732, "cost_per_test": 0.011672, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 64.706, "latency": 854.891, "stderr": 4.732, "cost_per_test": 4.288697, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 59.804, "latency": 240.609, "stderr": 4.855, "cost_per_test": 0.031792, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 57.843, "latency": 136.077, "stderr": 4.889, "cost_per_test": 0.106331, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "poolside/laguna-m.1": { "accuracy": 50.98, "latency": 1291.927, "stderr": 4.95, "cost_per_test": 0.565715, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "poolside/laguna-xs.2": { "accuracy": 43.137, "latency": 265.874, "stderr": 4.904, "cost_per_test": 0.184341, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null } }, "finance_agent": { "meta/muse_spark_1_2": { "accuracy": 59.422, "latency": 304.037, "stderr": 1.467, "cost_per_test": 0.725022, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 58.101, "latency": 316.094, "stderr": 0.874, "cost_per_test": 2.505488, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 57.109, "latency": 438.872, "stderr": 1.089, "cost_per_test": 0.71378, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 56.57, "latency": 563.724, "stderr": 0.443, "cost_per_test": 5.463988, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 56.349, "latency": 537.752, "stderr": 0.831, "cost_per_test": 8.061563, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 55.882, "latency": 825.918, "stderr": 2.31, "cost_per_test": 0.410909, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 55.555, "latency": 659.443, "stderr": 0.335, "cost_per_test": 0.241264, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 55.302, "latency": 295.319, "stderr": 1.29, "cost_per_test": 1.393912, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 54.955, "latency": 1104.076, "stderr": 0.439, "cost_per_test": 1.180225, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 53.949, "latency": 522.678, "stderr": 1.764, "cost_per_test": 4.218538, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 52.626, "latency": 388.501, "stderr": 0.537, "cost_per_test": 0.239646, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 51.979, "latency": 867.393, "stderr": 0.534, "cost_per_test": 4.021924, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 51.834, "latency": 629.768, "stderr": 0.461, "cost_per_test": 3.82068, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 51.45, "latency": 322.275, "stderr": 0.251, "cost_per_test": 3.638017, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 51.033, "latency": 629.1, "stderr": 1.076, "cost_per_test": 2.284299, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 50.83, "latency": 366.437, "stderr": 0.203, "cost_per_test": 0.662554, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 48.844, "latency": 457.07, "stderr": 1.299, "cost_per_test": 0.077067, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.5": { "accuracy": 48.751, "latency": 264.863, "stderr": 0.1, "cost_per_test": 1.119629, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 48.657, "latency": 1157.293, "stderr": 1.799, "cost_per_test": 1.181106, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 48.353, "latency": 483.912, "stderr": 0.477, "cost_per_test": 1.971421, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 48.008, "latency": 502.942, "stderr": 1.421, "cost_per_test": 0.291233, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 46.626, "latency": 200.51, "stderr": 1.071, "cost_per_test": 0.377362, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling": { "accuracy": 45.971, "latency": 1107.778, "stderr": 1.997, "cost_per_test": 1.189121, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 45.407, "latency": 2121.804, "stderr": 1.406, "cost_per_test": 1.162264, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 44.932, "latency": 1552.199, "stderr": 1.709, "cost_per_test": 1.097763, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.1": { "accuracy": 44.79, "latency": 894.632, "stderr": 0.347, "cost_per_test": 0.582253, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 44.081, "latency": 1489.313, "stderr": 0.767, "cost_per_test": 1.110116, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 42.968, "latency": 196.068, "stderr": 1.931, "cost_per_test": 1.378409, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 42.476, "latency": 241.608, "stderr": 0.666, "cost_per_test": 0.543671, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 42.438, "latency": 734.013, "stderr": 2.325, "cost_per_test": 0.224865, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 41.245, "latency": 639.699, "stderr": 1.036, "cost_per_test": 0.342801, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 41.243, "latency": 391.709, "stderr": 1.36, "cost_per_test": 0.190978, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 40.88, "latency": 601.297, "stderr": 0.729, "cost_per_test": 1.175366, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 38.236, "latency": 326.257, "stderr": 0.381, "cost_per_test": 0.152321, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 38.16, "latency": 239.629, "stderr": 2.661, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "grok/grok-4.3": { "accuracy": 37.766, "latency": 622.363, "stderr": 0.698, "cost_per_test": 0.842163, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 37.61, "latency": 267.315, "stderr": 1.198, "cost_per_test": 0.088638, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 31.511, "latency": 661.916, "stderr": 2.394, "cost_per_test": 1.736097, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 30.999, "latency": 180.056, "stderr": 0.088, "cost_per_test": 0.617074, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 30.068, "latency": 53.577, "stderr": 0.727, "cost_per_test": 0.133469, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 28.573, "latency": 351.293, "stderr": 0.598, "cost_per_test": 0.803345, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 28.532, "latency": 157.971, "stderr": 0.306, "cost_per_test": 0.049759, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 27.943, "latency": 750.548, "stderr": 1.058, "cost_per_test": 0.169425, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "poolside/laguna-m.1": { "accuracy": 24.634, "latency": 1359.272, "stderr": 1.074, "cost_per_test": 0.839467, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 18.693, "latency": 478.139, "stderr": 1.669, "cost_per_test": 0.213685, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } }, "terminal_bench_2_1": { "openai/gpt-5.6-sol": { "accuracy": 85.768, "latency": 372.819, "stderr": 1.35, "cost_per_test": 1.023319, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 84.644, "latency": 752.932, "stderr": 0.991, "cost_per_test": 0.886936, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 80.899, "latency": 608.575, "stderr": 0.649, "cost_per_test": 0.312662, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 80.524, "latency": 504.537, "stderr": 1.35, "cost_per_test": 1.429025, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 79.026, "latency": 355.751, "stderr": 0.991, "cost_per_test": 0.053767, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 76.404, "latency": 427.412, "stderr": 0.649, "cost_per_test": 0.741883, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 74.532, "latency": 635.043, "stderr": 2.085, "cost_per_test": 0.796027, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 74.157, "latency": 373.223, "stderr": 1.124, "cost_per_test": 0.868179, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 73.783, "latency": 552.153, "stderr": 1.633, "cost_per_test": 1.174404, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 73.408, "latency": 260.9, "stderr": 2.085, "cost_per_test": 0.284936, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 71.91, "latency": 929.902, "stderr": 0.649, "cost_per_test": 2.409772, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 70.787, "latency": 410.249, "stderr": 1.124, "cost_per_test": 0.582677, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 69.663, "latency": 879.273, "stderr": 0.649, "cost_per_test": 0.499472, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 69.288, "latency": 624.772, "stderr": 0.991, "cost_per_test": 0.417396, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 68.539, "latency": 816.587, "stderr": 1.297, "cost_per_test": 1.977087, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 67.79, "latency": 624.415, "stderr": 4.416, "cost_per_test": 0.283943, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.2": { "accuracy": 67.79, "latency": 820.506, "stderr": 0.991, "cost_per_test": 0.425184, "temperature": null, "top_p": 1.0, "max_output_tokens": 48000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 67.416, "latency": 890.917, "stderr": 0.649, "cost_per_test": 0.448084, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 67.041, "latency": 683.03, "stderr": 1.35, "cost_per_test": 0.021068, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 61.049, "latency": 935.196, "stderr": 0.375, "cost_per_test": 0.33233, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 60.674, "latency": 775.239, "stderr": 5.207, "cost_per_test": 0.021658, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 57.303, "latency": 704.225, "stderr": 0.649, "cost_per_test": 0.572392, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 57.303, "latency": 866.253, "stderr": 5.273, "cost_per_test": 0.043613, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.1": { "accuracy": 56.929, "latency": 940.943, "stderr": 1.35, "cost_per_test": 0.232386, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 55.056, "latency": 603.689, "stderr": 2.339, "cost_per_test": 0.127848, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 54.682, "latency": 673.036, "stderr": 4.556, "cost_per_test": 0.286739, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 53.933, "latency": 376.158, "stderr": 1.297, "cost_per_test": 0.154312, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 53.558, "latency": 932.358, "stderr": 2.085, "cost_per_test": 0.138449, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 53.558, "latency": 1023.416, "stderr": 0.749, "cost_per_test": 0.205973, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 53.184, "latency": 778.109, "stderr": 1.633, "cost_per_test": 0.201902, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 52.809, "latency": 654.983, "stderr": 0.649, "cost_per_test": 0.095849, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 50.936, "latency": 584.116, "stderr": 2.925, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 50.187, "latency": 416.383, "stderr": 0.991, "cost_per_test": 0.337052, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 50.187, "latency": 593.826, "stderr": 2.456, "cost_per_test": 0.067399, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 50.187, "latency": 1078.158, "stderr": 1.498, "cost_per_test": 0.239191, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 48.689, "latency": 812.638, "stderr": 3.066, "cost_per_test": 0.066012, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 47.566, "latency": 771.551, "stderr": 0.375, "cost_per_test": 0.630691, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 44.195, "latency": 305.726, "stderr": 0.991, "cost_per_test": 0.407362, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 43.82, "latency": 649.489, "stderr": 1.946, "cost_per_test": 0.363822, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 41.948, "latency": 470.713, "stderr": 2.622, "cost_per_test": 0.982203, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 41.573, "latency": 757.485, "stderr": 2.973, "cost_per_test": 0.072321, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 38.951, "latency": 989.169, "stderr": 2.701, "cost_per_test": 1.190984, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 34.082, "latency": 278.342, "stderr": 0.991, "cost_per_test": 0.084894, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 34.082, "latency": 1209.608, "stderr": 1.633, "cost_per_test": 0.300575, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 25.843, "latency": 1218.578, "stderr": 2.973, "cost_per_test": 0.253714, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } }, "corp_fin_v2": { "anthropic/claude-opus-5": { "accuracy": 74.942, "latency": 29.037, "stderr": 1.479, "cost_per_test": 0.696945, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 73.077, "latency": 25.409, "stderr": 1.514, "cost_per_test": 0.087456, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 72.727, "latency": 31.073, "stderr": 1.52, "cost_per_test": 0.086787, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 72.611, "latency": 30.106, "stderr": 1.522, "cost_per_test": 0.212074, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 71.562, "latency": 58.898, "stderr": 1.54, "cost_per_test": 1.737573, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 70.047, "latency": 24.725, "stderr": 1.564, "cost_per_test": 0.081293, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 69.814, "latency": 51.921, "stderr": 1.567, "cost_per_test": 0.021752, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 69.697, "latency": 23.548, "stderr": 1.569, "cost_per_test": 0.041574, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.5": { "accuracy": 69.58, "latency": 26.764, "stderr": 1.571, "cost_per_test": 0.43166, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 69.231, "latency": 45.083, "stderr": 1.576, "cost_per_test": 0.072205, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.5": { "accuracy": 68.415, "latency": 12.076, "stderr": 1.587, "cost_per_test": 0.131473, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 68.182, "latency": 96.974, "stderr": 1.59, "cost_per_test": 0.042064, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 67.949, "latency": 20.457, "stderr": 1.593, "cost_per_test": 0.416108, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 67.716, "latency": 14.833, "stderr": 1.596, "cost_per_test": 0.039017, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 67.366, "latency": 55.876, "stderr": 1.601, "cost_per_test": 0.146023, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 66.667, "latency": 20.037, "stderr": 1.609, "cost_per_test": 0.287644, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 66.434, "latency": 9.294, "stderr": 1.612, "cost_per_test": 0.138028, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 66.434, "latency": 20.315, "stderr": 1.612, "cost_per_test": 0.367275, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 66.317, "latency": 26.348, "stderr": 1.614, "cost_per_test": 0.0982, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 66.084, "latency": 18.221, "stderr": 1.616, "cost_per_test": 0.654716, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 65.967, "latency": 20.38, "stderr": 1.618, "cost_per_test": 0.679358, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 65.851, "latency": 9.418, "stderr": 1.619, "cost_per_test": 0.130871, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 65.734, "latency": 28.137, "stderr": 1.62, "cost_per_test": 0.145587, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 65.385, "latency": 18.336, "stderr": 1.624, "cost_per_test": 0.17462, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.1": { "accuracy": 65.268, "latency": 45.736, "stderr": 1.625, "cost_per_test": 0.071771, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 65.035, "latency": 20.103, "stderr": 1.628, "cost_per_test": 0.01626, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 64.918, "latency": 15.701, "stderr": 1.629, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 64.569, "latency": 8.593, "stderr": 1.633, "cost_per_test": 0.140208, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 63.753, "latency": 11.623, "stderr": 1.641, "cost_per_test": 0.023635, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 63.52, "latency": 6.733, "stderr": 1.643, "cost_per_test": 0.110554, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 63.17, "latency": 15.049, "stderr": 1.647, "cost_per_test": 0.096327, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 62.471, "latency": 16.623, "stderr": 1.653, "cost_per_test": 0.029339, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 62.238, "latency": 4.411, "stderr": 1.655, "cost_per_test": 0.023324, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 62.238, "latency": 12.856, "stderr": 1.655, "cost_per_test": 0.005258, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 62.238, "latency": 35.919, "stderr": 1.655, "cost_per_test": 0.038254, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 62.005, "latency": 17.433, "stderr": 1.657, "cost_per_test": 0.009434, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 62.005, "latency": 34.056, "stderr": 1.657, "cost_per_test": 0.028532, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 61.888, "latency": 74.027, "stderr": 1.658, "cost_per_test": 0.148855, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 61.072, "latency": 7.363, "stderr": 1.665, "cost_per_test": 0.012353, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 60.839, "latency": 4.708, "stderr": 1.666, "cost_per_test": 0.018404, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 60.256, "latency": 45.279, "stderr": 1.671, "cost_per_test": 0.068348, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 60.14, "latency": 19.334, "stderr": 1.672, "cost_per_test": 0.008752, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 59.907, "latency": 42.14, "stderr": 1.673, "cost_per_test": 0.127154, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 55.478, "latency": 272.771, "stderr": 1.697, "cost_per_test": 0.007103, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-m.1": { "accuracy": 55.245, "latency": 203.595, "stderr": 1.698, "cost_per_test": 0.014239, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } } } }