{ "metadata": { "benchmark": "Vals Multimodal Index", "slug": "vals_multimodal_index", "description": "Benchmark consisting of a weighted performance across finance, coding, and education tasks. Showing the potential impact that LLM's can have on the economy.", "benchmark_id": "vals_multimodal_index", "family": "vals_multimodal_index", "version": "1.2", "updated": "2026-08-11", "dataset_type": "private", "industry": "index", "tasks": { "overall": "Overall", "finance_agent": "Finance Agent v2", "corp_fin_v2": "CorpFin v2", "swebench": "SWE-bench", "terminal_bench_2_1": "Terminal-Bench 2.1", "vibe_code_bench": "Vibe Code Bench", "sage": "SAGE", "mortgage_tax": "Mortgage Tax" }, "models": [ "alibaba/qwen3.6-plus", "alibaba/qwen3.7-plus", "alibaba/qwen3.8-max", "anthropic/claude-fable-5", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.6-flash", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "minimax/MiniMax-M3", "mistralai/mistral-medium-3.5", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.5" ], "partners": [], "showBadge": true, "visible": true, "use_cost_per_test": true, "runner": "platform", "mode": "agentic", "archived": false, "partner": false, "total_models": 33 }, "tasks": { "overall": { "anthropic/claude-fable-5": { "accuracy": 74.154, "latency": 769.489, "stderr": 0.567, "cost_per_test": 7.972213, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 73.896, "latency": 853.761, "stderr": 1.243, "cost_per_test": 6.136905, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 73.42, "latency": 906.432, "stderr": 0.756, "cost_per_test": 1.696507, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 72.64, "latency": 550.773, "stderr": 0.676, "cost_per_test": 5.357844, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 70.889, "latency": 979.847, "stderr": 0.744, "cost_per_test": 5.455981, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 69.796, "latency": 445.979, "stderr": 0.989, "cost_per_test": 0.506189, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 69.058, "latency": 406.659, "stderr": 0.761, "cost_per_test": 0.160787, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 68.826, "latency": 1012.043, "stderr": 0.909, "cost_per_test": 6.532939, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 68.071, "latency": 503.747, "stderr": 1.419, "cost_per_test": 3.317135, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 67.361, "latency": 564.055, "stderr": 1.162, "cost_per_test": 4.44122, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 66.742, "latency": 288.194, "stderr": 1.151, "cost_per_test": 0.363107, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 65.394, "latency": 1996.162, "stderr": 1.449, "cost_per_test": 1.926436, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 65.077, "latency": 535.655, "stderr": 1.353, "cost_per_test": 1.023831, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 65.07, "latency": 258.088, "stderr": 1.571, "cost_per_test": 0.800829, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 63.42, "latency": 318.501, "stderr": 1.292, "cost_per_test": 0.910544, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 62.928, "latency": 266.485, "stderr": 1.464, "cost_per_test": 1.011224, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 60.571, "latency": 536.038, "stderr": 1.385, "cost_per_test": 1.576917, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 59.974, "latency": 1076.237, "stderr": 1.646, "cost_per_test": 1.076447, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 56.434, "latency": 903.811, "stderr": 1.735, "cost_per_test": 0.517632, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 56.068, "latency": 322.907, "stderr": 1.406, "cost_per_test": 0.977777, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 54.219, "latency": 788.336, "stderr": 1.863, "cost_per_test": 0.488419, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 53.892, "latency": 628.662, "stderr": 1.478, "cost_per_test": 0.255213, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 52.767, "latency": 430.314, "stderr": 1.419, "cost_per_test": 0.029291, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 52.187, "latency": 260.281, "stderr": 1.414, "cost_per_test": 0.28728, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 51.516, "latency": 642.883, "stderr": 1.327, "cost_per_test": 1.117056, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 50.053, "latency": 709.047, "stderr": 1.269, "cost_per_test": 0.139289, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 49.403, "latency": 929.083, "stderr": 1.115, "cost_per_test": 0.632327, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 47.643, "latency": 671.046, "stderr": 1.707, "cost_per_test": 0.232894, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 43.293, "latency": 339.036, "stderr": 1.236, "cost_per_test": 0.49718, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 42.883, "latency": 282.986, "stderr": 0.932, "cost_per_test": 0.409227, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 41.352, "latency": 152.434, "stderr": 0.661, "cost_per_test": 0.123381, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 39.059, "latency": 170.988, "stderr": 0.701, "cost_per_test": 0.398244, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 34.768, "latency": 734.703, "stderr": 0.833, "cost_per_test": 3.453606, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null } }, "finance_agent": { "meta/muse_spark_1_2": { "accuracy": 59.422, "latency": 304.037, "stderr": 1.467, "cost_per_test": 0.725022, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 58.101, "latency": 316.094, "stderr": 0.874, "cost_per_test": 2.505488, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 57.109, "latency": 438.872, "stderr": 1.089, "cost_per_test": 0.71378, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 56.57, "latency": 563.724, "stderr": 0.443, "cost_per_test": 5.463988, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 56.349, "latency": 537.752, "stderr": 0.831, "cost_per_test": 8.061563, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 55.882, "latency": 825.918, "stderr": 2.31, "cost_per_test": 0.410909, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 55.555, "latency": 659.443, "stderr": 0.335, "cost_per_test": 0.241264, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 55.302, "latency": 295.319, "stderr": 1.29, "cost_per_test": 1.393912, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 54.955, "latency": 1104.076, "stderr": 0.439, "cost_per_test": 1.180225, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 53.949, "latency": 522.678, "stderr": 1.764, "cost_per_test": 4.218538, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 52.626, "latency": 388.501, "stderr": 0.537, "cost_per_test": 0.239646, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 51.979, "latency": 867.393, "stderr": 0.534, "cost_per_test": 4.021924, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 51.834, "latency": 629.768, "stderr": 0.461, "cost_per_test": 3.82068, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 51.45, "latency": 322.275, "stderr": 0.251, "cost_per_test": 3.638017, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 51.033, "latency": 629.1, "stderr": 1.076, "cost_per_test": 2.284299, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 48.751, "latency": 264.863, "stderr": 0.1, "cost_per_test": 1.119629, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 48.657, "latency": 1157.293, "stderr": 1.799, "cost_per_test": 1.181106, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 48.008, "latency": 502.942, "stderr": 1.421, "cost_per_test": 0.291233, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 45.971, "latency": 1107.778, "stderr": 1.997, "cost_per_test": 1.189121, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 45.407, "latency": 2121.804, "stderr": 1.406, "cost_per_test": 1.162264, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 44.932, "latency": 1552.199, "stderr": 1.709, "cost_per_test": 1.097763, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 42.968, "latency": 196.068, "stderr": 1.931, "cost_per_test": 1.378409, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 42.476, "latency": 241.608, "stderr": 0.666, "cost_per_test": 0.543671, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 42.438, "latency": 734.013, "stderr": 2.325, "cost_per_test": 0.224865, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 41.245, "latency": 639.699, "stderr": 1.036, "cost_per_test": 0.342801, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 40.88, "latency": 601.297, "stderr": 0.729, "cost_per_test": 1.175366, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 38.236, "latency": 326.257, "stderr": 0.381, "cost_per_test": 0.152321, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 37.766, "latency": 622.363, "stderr": 0.698, "cost_per_test": 0.842163, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 37.61, "latency": 267.315, "stderr": 1.198, "cost_per_test": 0.088638, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 31.511, "latency": 661.916, "stderr": 2.394, "cost_per_test": 1.736097, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 30.999, "latency": 180.056, "stderr": 0.088, "cost_per_test": 0.617074, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 30.068, "latency": 53.577, "stderr": 0.727, "cost_per_test": 0.133469, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 28.573, "latency": 351.293, "stderr": 0.598, "cost_per_test": 0.803345, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } }, "corp_fin_v2": { "anthropic/claude-opus-5": { "accuracy": 74.942, "latency": 29.037, "stderr": 1.479, "cost_per_test": 0.696945, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 73.077, "latency": 25.409, "stderr": 1.514, "cost_per_test": 0.087456, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 72.727, "latency": 31.073, "stderr": 1.52, "cost_per_test": 0.086787, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 72.611, "latency": 30.106, "stderr": 1.522, "cost_per_test": 0.212074, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 71.562, "latency": 58.898, "stderr": 1.54, "cost_per_test": 1.737573, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 70.047, "latency": 24.725, "stderr": 1.564, "cost_per_test": 0.081293, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 69.814, "latency": 51.921, "stderr": 1.567, "cost_per_test": 0.021752, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 69.697, "latency": 23.548, "stderr": 1.569, "cost_per_test": 0.041574, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.5": { "accuracy": 69.58, "latency": 26.764, "stderr": 1.571, "cost_per_test": 0.43166, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 69.231, "latency": 45.083, "stderr": 1.576, "cost_per_test": 0.072205, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.5": { "accuracy": 68.415, "latency": 12.076, "stderr": 1.587, "cost_per_test": 0.131473, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 68.182, "latency": 96.974, "stderr": 1.59, "cost_per_test": 0.042064, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 67.949, "latency": 20.457, "stderr": 1.593, "cost_per_test": 0.416108, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 67.716, "latency": 14.833, "stderr": 1.596, "cost_per_test": 0.039017, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 67.366, "latency": 55.876, "stderr": 1.601, "cost_per_test": 0.146023, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 66.667, "latency": 20.037, "stderr": 1.609, "cost_per_test": 0.287644, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 66.434, "latency": 9.294, "stderr": 1.612, "cost_per_test": 0.138028, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 66.434, "latency": 20.315, "stderr": 1.612, "cost_per_test": 0.367275, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 66.084, "latency": 18.221, "stderr": 1.616, "cost_per_test": 0.654716, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 65.967, "latency": 20.38, "stderr": 1.618, "cost_per_test": 0.679358, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 65.851, "latency": 9.418, "stderr": 1.619, "cost_per_test": 0.130871, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 65.734, "latency": 28.137, "stderr": 1.62, "cost_per_test": 0.145587, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 65.035, "latency": 20.103, "stderr": 1.628, "cost_per_test": 0.01626, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 64.569, "latency": 8.593, "stderr": 1.633, "cost_per_test": 0.140208, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 63.52, "latency": 6.733, "stderr": 1.643, "cost_per_test": 0.110554, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 63.17, "latency": 15.049, "stderr": 1.647, "cost_per_test": 0.096327, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 62.238, "latency": 35.919, "stderr": 1.655, "cost_per_test": 0.038254, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 62.005, "latency": 34.056, "stderr": 1.657, "cost_per_test": 0.028532, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 61.072, "latency": 7.363, "stderr": 1.665, "cost_per_test": 0.012353, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 60.839, "latency": 4.708, "stderr": 1.666, "cost_per_test": 0.018404, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 60.256, "latency": 45.279, "stderr": 1.671, "cost_per_test": 0.068348, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 60.14, "latency": 19.334, "stderr": 1.672, "cost_per_test": 0.008752, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 59.907, "latency": 42.14, "stderr": 1.673, "cost_per_test": 0.127154, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null } }, "swebench": { "openai/gpt-5.6-sol": { "accuracy": 99.02, "latency": 237.142, "stderr": 0.976, "cost_per_test": 1.311279, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "anthropic/claude-fable-5": { "accuracy": 97.059, "latency": 441.198, "stderr": 1.673, "cost_per_test": 2.046736, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "kimi/kimi-k3": { "accuracy": 95.098, "latency": 611.386, "stderr": 2.138, "cost_per_test": 0.879924, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "anthropic/claude-opus-5": { "accuracy": 95.098, "latency": 665.89, "stderr": 2.138, "cost_per_test": 1.554764, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 93.137, "latency": 184.771, "stderr": 2.503, "cost_per_test": 0.051004, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 89.216, "latency": 630.305, "stderr": 3.071, "cost_per_test": 1.923224, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 87.255, "latency": 1553.607, "stderr": 3.302, "cost_per_test": 1.577912, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "grok/grok-4.5": { "accuracy": 86.275, "latency": 210.846, "stderr": 3.407, "cost_per_test": 0.606056, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "meta/muse_spark_1_2": { "accuracy": 85.294, "latency": 638.55, "stderr": 3.507, "cost_per_test": 0.644011, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openai/gpt-5.5": { "accuracy": 82.6, "latency": 426.425, "stderr": 1.695, "cost_per_test": 1.361765, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 82.353, "latency": 508.652, "stderr": 3.775, "cost_per_test": 2.999546, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "meta/muse_spark_1_1": { "accuracy": 78.431, "latency": 238.11, "stderr": 4.072, "cost_per_test": 0.352022, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 77.451, "latency": 633.682, "stderr": 4.138, "cost_per_test": 1.706873, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "google/gemini-3.6-flash": { "accuracy": 77.451, "latency": 1340.979, "stderr": 4.138, "cost_per_test": 1.380993, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "google/gemini-3.5-flash": { "accuracy": 75.49, "latency": 230.39, "stderr": 4.259, "cost_per_test": 0.953882, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 75.49, "latency": 305.548, "stderr": 4.259, "cost_per_test": 0.13413, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 75.49, "latency": 1097.115, "stderr": 4.259, "cost_per_test": 1.749654, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "thinkingmachines/inkling": { "accuracy": 75.49, "latency": 1420.277, "stderr": 4.259, "cost_per_test": 0.585966, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "kimi/kimi-k2.6": { "accuracy": 74.51, "latency": 396.661, "stderr": 4.315, "cost_per_test": 0.322908, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 73.529, "latency": 142.056, "stderr": 4.368, "cost_per_test": 0.286403, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 72.549, "latency": 329.202, "stderr": 4.419, "cost_per_test": 0.820116, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "grok/grok-4.3": { "accuracy": 70.588, "latency": 126.579, "stderr": 4.512, "cost_per_test": 0.23972, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 70.588, "latency": 487.435, "stderr": 4.512, "cost_per_test": 0.901327, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "minimax/MiniMax-M3": { "accuracy": 69.608, "latency": 765.232, "stderr": 4.554, "cost_per_test": 0.508278, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 68.627, "latency": 327.172, "stderr": 4.594, "cost_per_test": 0.306186, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 67.647, "latency": 281.133, "stderr": 4.632, "cost_per_test": 0.107316, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 67.647, "latency": 355.599, "stderr": 4.632, "cost_per_test": 0.589598, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 66.667, "latency": 151.343, "stderr": 4.668, "cost_per_test": 0.586871, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 66.667, "latency": 602.556, "stderr": 4.668, "cost_per_test": 0.207597, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 64.706, "latency": 264.186, "stderr": 4.732, "cost_per_test": 0.407017, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 64.706, "latency": 284.365, "stderr": 4.732, "cost_per_test": 0.011672, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 64.706, "latency": 854.891, "stderr": 4.732, "cost_per_test": 4.288697, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 57.843, "latency": 136.077, "stderr": 4.889, "cost_per_test": 0.106331, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null, "harness": null } }, "sage": { "anthropic/claude-opus-4-7": { "accuracy": 56.103, "latency": 124.647, "stderr": 3.368, "cost_per_test": 0.327557, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 54.788, "latency": 185.594, "stderr": 3.347, "cost_per_test": 0.47466, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 54.26, "latency": 148.595, "stderr": 3.425, "cost_per_test": 0.116416, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 52.563, "latency": 75.395, "stderr": 3.416, "cost_per_test": 0.147977, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 51.89, "latency": 116.916, "stderr": 3.4, "cost_per_test": 0.618162, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 51.849, "latency": 42.697, "stderr": 3.404, "cost_per_test": 0.018229, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 51.532, "latency": 76.05, "stderr": 3.954, "cost_per_test": 0.130637, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 51.254, "latency": 132.802, "stderr": 3.415, "cost_per_test": 0.073645, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 50.813, "latency": 141.066, "stderr": 3.403, "cost_per_test": 0.075919, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 50.574, "latency": 118.538, "stderr": 3.436, "cost_per_test": 0.025844, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 50.224, "latency": 270.956, "stderr": 3.429, "cost_per_test": 0.068198, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 49.885, "latency": 32.592, "stderr": 3.438, "cost_per_test": 0.052555, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 49.54, "latency": 16.257, "stderr": 3.484, "cost_per_test": 0.00773, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 49.426, "latency": 56.168, "stderr": 3.291, "cost_per_test": 0.192726, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 48.925, "latency": 434.2, "stderr": 3.405, "cost_per_test": 0.588528, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 48.686, "latency": 21.96, "stderr": 3.383, "cost_per_test": 0.045301, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 48.677, "latency": 61.174, "stderr": 3.293, "cost_per_test": 0.068908, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 47.659, "latency": 29.074, "stderr": 3.44, "cost_per_test": 0.032261, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 47.003, "latency": 40.057, "stderr": 3.403, "cost_per_test": 0.03724, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 46.582, "latency": 163.365, "stderr": 3.43, "cost_per_test": 0.234647, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 45.58, "latency": 49.948, "stderr": 3.276, "cost_per_test": 0.025502, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 44.86, "latency": 229.267, "stderr": 3.433, "cost_per_test": 0.041429, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 44.22, "latency": 51.878, "stderr": 3.331, "cost_per_test": 0.01037, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 43.265, "latency": 52.66, "stderr": 3.387, "cost_per_test": 0.004398, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 39.251, "latency": 177.044, "stderr": 3.384, "cost_per_test": 0.021774, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 38.242, "latency": 66.358, "stderr": 3.425, "cost_per_test": 0.063231, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 38.081, "latency": 34.334, "stderr": 3.087, "cost_per_test": 0.006061, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 37.613, "latency": 55.844, "stderr": 3.356, "cost_per_test": 0.072775, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 36.555, "latency": 198.023, "stderr": 3.216, "cost_per_test": 0.163151, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.5": { "accuracy": 34.967, "latency": 169.791, "stderr": 3.316, "cost_per_test": 0.115513, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 32.744, "latency": 206.475, "stderr": 3.258, "cost_per_test": 0.022636, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 31.822, "latency": 66.445, "stderr": 3.134, "cost_per_test": 0.05313, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 19.736, "latency": 96.2, "stderr": 2.647, "cost_per_test": 0.037978, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } }, "mortgage_tax": { "anthropic/claude-opus-5": { "accuracy": 72.059, "latency": 7.978, "stderr": 0.875, "cost_per_test": 0.07375, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 70.27, "latency": 16.101, "stderr": 0.896, "cost_per_test": 0.084417, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 70.032, "latency": 28.668, "stderr": 0.897, "cost_per_test": 0.080204, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 69.912, "latency": 20.814, "stderr": 0.893, "cost_per_test": 0.105627, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 69.396, "latency": 23.616, "stderr": 0.908, "cost_per_test": 0.023542, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 68.919, "latency": 16.096, "stderr": 0.909, "cost_per_test": 0.175013, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 68.76, "latency": 28.243, "stderr": 0.912, "cost_per_test": 0.071407, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 68.72, "latency": 12.838, "stderr": 0.914, "cost_per_test": 0.007477, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 68.362, "latency": 26.756, "stderr": 0.91, "cost_per_test": 0.008326, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 68.124, "latency": 11.428, "stderr": 0.91, "cost_per_test": 0.017873, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 68.044, "latency": 5.368, "stderr": 0.912, "cost_per_test": 0.00284, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 67.965, "latency": 78.18, "stderr": 0.908, "cost_per_test": 0.013476, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 67.726, "latency": 29.74, "stderr": 0.921, "cost_per_test": 0.04467, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 67.608, "latency": 9.239, "stderr": 0.915, "cost_per_test": 0.019658, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 67.329, "latency": 6.316, "stderr": 0.925, "cost_per_test": 0.022706, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 67.29, "latency": 22.835, "stderr": 0.921, "cost_per_test": 0.004804, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 67.289, "latency": 12.618, "stderr": 0.922, "cost_per_test": 0.071198, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 66.336, "latency": 78.943, "stderr": 0.909, "cost_per_test": 0.038981, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 66.176, "latency": 60.548, "stderr": 0.928, "cost_per_test": 0.007588, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 66.136, "latency": 24.818, "stderr": 0.928, "cost_per_test": 0.022312, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 65.818, "latency": 109.76, "stderr": 0.93, "cost_per_test": 0.02869, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 65.422, "latency": 44.623, "stderr": 0.934, "cost_per_test": 0.026128, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 63.99, "latency": 47.499, "stderr": 0.942, "cost_per_test": 0.027349, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 63.99, "latency": 69.108, "stderr": 0.944, "cost_per_test": 0.056492, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 63.514, "latency": 126.454, "stderr": 0.906, "cost_per_test": 0.044769, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 62.282, "latency": 112.214, "stderr": 0.952, "cost_per_test": 0.011526, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 62.162, "latency": 29.792, "stderr": 0.962, "cost_per_test": 0.020742, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 61.804, "latency": 117.461, "stderr": 0.968, "cost_per_test": 0.014998, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 59.26, "latency": 32.939, "stderr": 0.979, "cost_per_test": 0.001523, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 59.102, "latency": 14.526, "stderr": 0.964, "cost_per_test": 0.002585, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 48.252, "latency": 443.266, "stderr": 0.973, "cost_per_test": 0.012184, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 45.35, "latency": 11.124, "stderr": 0.988, "cost_per_test": 0.024144, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 28.895, "latency": 68.43, "stderr": 0.838, "cost_per_test": 0.024032, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null } }, "vibe_code_bench": { "anthropic/claude-fable-5": { "accuracy": 93.467, "latency": 3711.026, "stderr": 1.473, "cost_per_test": 41.737416, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 91.271, "latency": 4041.504, "stderr": 2.21, "cost_per_test": 9.904582, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 89.869, "latency": 4549.256, "stderr": 2.244, "cost_per_test": 28.380692, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 89.34, "latency": 2033.047, "stderr": 2.575, "cost_per_test": 33.403634, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 86.902, "latency": 4001.424, "stderr": 3.468, "cost_per_test": 38.07813, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 85.664, "latency": 3900.597, "stderr": 6.078, "cost_per_test": 34.089223, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 82.854, "latency": 1195.221, "stderr": 4.137, "cost_per_test": 1.529642, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 81.053, "latency": 1551.831, "stderr": 3.009, "cost_per_test": 0.748037, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 77.701, "latency": 2141.902, "stderr": 5.336, "cost_per_test": 21.407199, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 77.099, "latency": 1911.567, "stderr": 7.028, "cost_per_test": 16.661911, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 74.911, "latency": 10135.139, "stderr": 6.9, "cost_per_test": 10.030932, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 72.818, "latency": 959.495, "stderr": 7.59, "cost_per_test": 4.596845, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 72.754, "latency": 830.057, "stderr": 5.807, "cost_per_test": 4.102194, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 72.412, "latency": 615.428, "stderr": 5.168, "cost_per_test": 0.923283, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 69.941, "latency": 1523.201, "stderr": 6.273, "cost_per_test": 3.041993, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 56.54, "latency": 5073.23, "stderr": 7.963, "cost_per_test": 6.453899, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 56.223, "latency": 1572.118, "stderr": 6.546, "cost_per_test": 5.907892, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 54.709, "latency": 893.074, "stderr": 7.002, "cost_per_test": 2.540385, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 48.148, "latency": 1580.348, "stderr": 6.086, "cost_per_test": 0.068395, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 46.94, "latency": 2231.75, "stderr": 7.011, "cost_per_test": 1.082348, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 42.523, "latency": 2055.117, "stderr": 8.901, "cost_per_test": 1.191299, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 42.323, "latency": 2967.768, "stderr": 8.392, "cost_per_test": 1.925351, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 34.214, "latency": 1211.906, "stderr": 6.444, "cost_per_test": 3.825202, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 30.032, "latency": 3276.224, "stderr": 8.282, "cost_per_test": 1.277298, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 28.253, "latency": 2289.977, "stderr": 6.126, "cost_per_test": 5.447636, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 21.954, "latency": 806.66, "stderr": 6.673, "cost_per_test": 0.942067, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 19.039, "latency": 2949.469, "stderr": 5.403, "cost_per_test": 0.432264, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.3": { "accuracy": 15.475, "latency": 589.407, "stderr": 5.576, "cost_per_test": 1.28472, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 13.277, "latency": 2891.761, "stderr": 4.646, "cost_per_test": 1.728665, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 9.882, "latency": 775.882, "stderr": 3.549, "cost_per_test": 1.306474, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 1.515, "latency": 301.651, "stderr": 1.515, "cost_per_test": 0.771885, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 1.24, "latency": 2470.533, "stderr": 1.24, "cost_per_test": 16.735503, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 572.711, "stderr": 0.0, "cost_per_test": 0.51, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "terminal_bench_2_1": { "openai/gpt-5.6-sol": { "accuracy": 85.768, "latency": 372.819, "stderr": 1.35, "cost_per_test": 1.023319, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 84.644, "latency": 752.932, "stderr": 0.991, "cost_per_test": 0.886936, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 80.899, "latency": 608.575, "stderr": 0.649, "cost_per_test": 0.312662, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 80.524, "latency": 504.537, "stderr": 1.35, "cost_per_test": 1.429025, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 79.026, "latency": 355.751, "stderr": 0.991, "cost_per_test": 0.053767, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 76.404, "latency": 427.412, "stderr": 0.649, "cost_per_test": 0.741883, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 74.532, "latency": 635.043, "stderr": 2.085, "cost_per_test": 0.796027, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 74.157, "latency": 373.223, "stderr": 1.124, "cost_per_test": 0.868179, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 73.783, "latency": 552.153, "stderr": 1.633, "cost_per_test": 1.174404, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 73.408, "latency": 260.9, "stderr": 2.085, "cost_per_test": 0.284936, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 71.91, "latency": 929.902, "stderr": 0.649, "cost_per_test": 2.409772, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 70.787, "latency": 410.249, "stderr": 1.124, "cost_per_test": 0.582677, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 69.663, "latency": 879.273, "stderr": 0.649, "cost_per_test": 0.499472, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 69.288, "latency": 624.772, "stderr": 0.991, "cost_per_test": 0.417396, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 68.539, "latency": 816.587, "stderr": 1.297, "cost_per_test": 1.977087, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 67.79, "latency": 624.415, "stderr": 4.416, "cost_per_test": 0.283943, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 67.416, "latency": 890.917, "stderr": 0.649, "cost_per_test": 0.448084, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 60.674, "latency": 775.239, "stderr": 5.207, "cost_per_test": 0.021658, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 57.303, "latency": 704.225, "stderr": 0.649, "cost_per_test": 0.572392, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 55.056, "latency": 603.689, "stderr": 2.339, "cost_per_test": 0.127848, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 54.682, "latency": 673.036, "stderr": 4.556, "cost_per_test": 0.286739, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 53.933, "latency": 376.158, "stderr": 1.297, "cost_per_test": 0.154312, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 53.558, "latency": 932.358, "stderr": 2.085, "cost_per_test": 0.138449, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 53.558, "latency": 1023.416, "stderr": 0.749, "cost_per_test": 0.205973, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 53.184, "latency": 778.109, "stderr": 1.633, "cost_per_test": 0.201902, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 52.809, "latency": 654.983, "stderr": 0.649, "cost_per_test": 0.095849, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 47.566, "latency": 771.551, "stderr": 0.375, "cost_per_test": 0.630691, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 44.195, "latency": 305.726, "stderr": 0.991, "cost_per_test": 0.407362, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 43.82, "latency": 649.489, "stderr": 1.946, "cost_per_test": 0.363822, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 41.948, "latency": 470.713, "stderr": 2.622, "cost_per_test": 0.982203, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 41.573, "latency": 757.485, "stderr": 2.973, "cost_per_test": 0.072321, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 38.951, "latency": 989.169, "stderr": 2.701, "cost_per_test": 1.190984, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 34.082, "latency": 278.342, "stderr": 0.991, "cost_per_test": 0.084894, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } } } }