{ "metadata": { "benchmark": "Vals Multimodal Index", "slug": "vals_multimodal_index_v1_1", "description": "Benchmark consisting of a weighted performance across finance, coding, and education tasks. Showing the potential impact that LLM's can have on the economy.", "benchmark_id": "vals_multimodal_index_v1_1", "family": "vals_multimodal_index", "version": "1.1", "updated": "2026-05-28", "dataset_type": "private", "industry": "index", "tasks": { "overall": "Overall", "finance_agent": "Finance Agent v2", "corp_fin_v2": "CorpFin v2", "swebench": "SWE-bench", "terminal_bench_2": "Terminal-Bench 2.0", "vibe_code_bench": "Vibe Code Bench", "sage": "SAGE", "mortgage_tax": "Mortgage Tax" }, "models": [ "alibaba/qwen3.6-plus", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-sonnet-4-6", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "kimi/kimi-k2.6", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5" ], "partners": [], "showBadge": true, "visible": true, "use_cost_per_test": true, "runner": "platform", "mode": "agentic", "archived": true, "total_models": 15 }, "tasks": { "overall": { "anthropic/claude-opus-4-8": { "accuracy": 70.712, "latency": 914.496, "stderr": 0.753, "cost_per_test": 1.94797, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "openai/gpt-5.5": { "accuracy": 67.768, "latency": 769.119, "stderr": 1.467, "cost_per_test": 3.468069, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-7": { "accuracy": 67.361, "latency": 541.813, "stderr": 1.246, "cost_per_test": 4.332646, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic" }, "google/gemini-3.5-flash": { "accuracy": 62.291, "latency": 271.527, "stderr": 1.466, "cost_per_test": 1.057689, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-sonnet-4-6": { "accuracy": 60.783, "latency": 523.363, "stderr": 1.469, "cost_per_test": 1.569284, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "kimi/kimi-k2.6": { "accuracy": 56.788, "latency": 880.829, "stderr": 1.794, "cost_per_test": 0.523246, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "google/gemini-3.1-pro-preview": { "accuracy": 55.749, "latency": 341.856, "stderr": 1.479, "cost_per_test": 0.968264, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 53.298, "latency": 983.419, "stderr": 1.862, "cost_per_test": 0.560175, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3-flash-preview": { "accuracy": 51.975, "latency": 271.697, "stderr": 1.496, "cost_per_test": 0.289336, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3.6-plus": { "accuracy": 50.737, "latency": 649.521, "stderr": 1.41, "cost_per_test": 1.031231, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 47.484, "latency": 923.967, "stderr": 1.736, "cost_per_test": 0.255302, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4.3": { "accuracy": 43.435, "latency": 551.052, "stderr": 1.281, "cost_per_test": 0.516233, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 42.352, "latency": 283.779, "stderr": 1.036, "cost_per_test": 0.404069, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 40.466, "latency": 208.082, "stderr": 0.786, "cost_per_test": 0.121223, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 38.704, "latency": 150.443, "stderr": 0.853, "cost_per_test": 0.362289, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" } }, "finance_agent": { "google/gemini-3.5-flash": { "accuracy": 58.101, "latency": 316.567, "stderr": 0.874, "cost_per_test": 2.505488, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-8": { "accuracy": 53.949, "latency": 522.678, "stderr": 1.764, "cost_per_test": 4.218538, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "openai/gpt-5.5": { "accuracy": 51.834, "latency": 616.56, "stderr": 0.461, "cost_per_test": 3.784562, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-7": { "accuracy": 51.45, "latency": 322.275, "stderr": 0.251, "cost_per_test": 3.638017, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic" }, "anthropic/claude-sonnet-4-6": { "accuracy": 51.033, "latency": 629.1, "stderr": 1.076, "cost_per_test": 2.284299, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 45.407, "latency": 1918.861, "stderr": 1.406, "cost_per_test": 1.037185, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2.6": { "accuracy": 44.932, "latency": 1528.223, "stderr": 1.709, "cost_per_test": 1.082722, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "google/gemini-3.1-pro-preview": { "accuracy": 42.968, "latency": 196.068, "stderr": 1.931, "cost_per_test": 1.378409, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3-flash-preview": { "accuracy": 42.476, "latency": 241.608, "stderr": 0.666, "cost_per_test": 0.543671, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3.6-plus": { "accuracy": 40.88, "latency": 585.203, "stderr": 0.729, "cost_per_test": 0.630525, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 38.236, "latency": 326.257, "stderr": 0.381, "cost_per_test": 0.152321, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4.3": { "accuracy": 37.766, "latency": 622.363, "stderr": 0.698, "cost_per_test": 0.842163, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 30.999, "latency": 180.056, "stderr": 0.088, "cost_per_test": 0.617074, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 30.068, "latency": 53.577, "stderr": 0.727, "cost_per_test": 0.133469, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 28.573, "latency": 351.797, "stderr": 0.598, "cost_per_test": 0.804911, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" } }, "corp_fin_v2": { "grok/grok-4.3": { "accuracy": 70.047, "latency": 24.725, "stderr": 1.564, "cost_per_test": 0.081293, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5.5": { "accuracy": 69.58, "latency": 26.764, "stderr": 1.571, "cost_per_test": 0.43166, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2.6": { "accuracy": 68.182, "latency": 96.974, "stderr": 1.59, "cost_per_test": 0.042064, "temperature": null, "top_p": 0.95, "max_output_tokens": 48000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "google/gemini-3-flash-preview": { "accuracy": 67.716, "latency": 14.833, "stderr": 1.596, "cost_per_test": 0.039017, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-sonnet-4-6": { "accuracy": 66.667, "latency": 20.037, "stderr": 1.609, "cost_per_test": 0.287644, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "anthropic/claude-opus-4-7": { "accuracy": 66.084, "latency": 18.221, "stderr": 1.616, "cost_per_test": 0.816017, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "anthropic/claude-opus-4-8": { "accuracy": 65.967, "latency": 20.38, "stderr": 1.618, "cost_per_test": 0.855534, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 65.851, "latency": 9.418, "stderr": 1.619, "cost_per_test": 0.130871, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-3.1-pro-preview": { "accuracy": 65.734, "latency": 28.137, "stderr": 1.62, "cost_per_test": 0.145587, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3.5-flash": { "accuracy": 64.569, "latency": 8.593, "stderr": 1.633, "cost_per_test": 0.140208, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 63.17, "latency": 15.049, "stderr": 1.647, "cost_per_test": 0.096327, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "alibaba/qwen3.6-plus": { "accuracy": 62.238, "latency": 35.919, "stderr": 1.655, "cost_per_test": 0.038254, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 61.072, "latency": 7.363, "stderr": 1.665, "cost_per_test": 0.01612, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 60.839, "latency": 4.708, "stderr": 1.666, "cost_per_test": 0.022927, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 60.256, "latency": 45.279, "stderr": 1.671, "cost_per_test": 0.068348, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" } }, "swebench": { "anthropic/claude-opus-4-8": { "accuracy": 89.216, "latency": 630.305, "stderr": 3.071, "cost_per_test": 1.923224, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-5.5": { "accuracy": 82.6, "latency": 426.425, "stderr": 1.695, "cost_per_test": 1.361765, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-opus-4-7": { "accuracy": 82.353, "latency": 508.652, "stderr": 3.775, "cost_per_test": 2.999546, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 77.451, "latency": 633.682, "stderr": 4.138, "cost_per_test": 1.706873, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "google/gemini-3.5-flash": { "accuracy": 75.49, "latency": 230.39, "stderr": 4.259, "cost_per_test": 0.953882, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "kimi/kimi-k2.6": { "accuracy": 74.51, "latency": 396.661, "stderr": 4.315, "cost_per_test": 0.322908, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "google/gemini-3.1-pro-preview": { "accuracy": 72.549, "latency": 329.202, "stderr": 4.419, "cost_per_test": 0.820116, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "grok/grok-4.3": { "accuracy": 70.588, "latency": 126.579, "stderr": 4.512, "cost_per_test": 0.23972, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "alibaba/qwen3.6-plus": { "accuracy": 70.588, "latency": 487.435, "stderr": 4.512, "cost_per_test": 0.901327, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "google/gemini-3-flash-preview": { "accuracy": 68.627, "latency": 327.172, "stderr": 4.594, "cost_per_test": 0.306186, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 67.647, "latency": 281.133, "stderr": 4.632, "cost_per_test": 0.107316, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 67.647, "latency": 355.599, "stderr": 4.632, "cost_per_test": 0.589598, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 66.667, "latency": 151.343, "stderr": 4.668, "cost_per_test": 0.586871, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 64.706, "latency": 264.186, "stderr": 4.732, "cost_per_test": 0.407017, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 57.843, "latency": 136.077, "stderr": 4.889, "cost_per_test": 0.106331, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null } }, "terminal_bench_2": { "openai/gpt-5.5": { "accuracy": 73.202, "latency": 2298.224, "stderr": 3.978, "cost_per_test": 1.819086, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-8": { "accuracy": 70.037, "latency": 472.445, "stderr": 1.35, "cost_per_test": 0.883864, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "anthropic/claude-opus-4-7": { "accuracy": 68.539, "latency": 660.89, "stderr": 4.95, "cost_per_test": 0.975907, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic" }, "google/gemini-3.5-flash": { "accuracy": 67.416, "latency": 408.048, "stderr": 1.297, "cost_per_test": 1.192081, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3.1-pro-preview": { "accuracy": 67.416, "latency": 542.887, "stderr": 4.996, "cost_per_test": 0.516084, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-sonnet-4-6": { "accuracy": 59.551, "latency": 615.5, "stderr": 5.232, "cost_per_test": 0.518966, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "kimi/kimi-k2.6": { "accuracy": 57.303, "latency": 795.458, "stderr": 5.273, "cost_per_test": 0.192786, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "google/gemini-3-flash-preview": { "accuracy": 51.685, "latency": 456.071, "stderr": 5.327, "cost_per_test": 0.168702, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3.6-plus": { "accuracy": 44.944, "latency": 840.666, "stderr": 5.303, "cost_per_test": 0.14597, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 44.944, "latency": 2241.556, "stderr": 4.492, "cost_per_test": 0.914107, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4.3": { "accuracy": 43.446, "latency": 1954.824, "stderr": 4.406, "cost_per_test": 1.115571, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 40.449, "latency": 161.408, "stderr": 5.232, "cost_per_test": 0.154109, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 39.888, "latency": 2527.932, "stderr": 4.454, "cost_per_test": 0.224388, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 38.202, "latency": 655.044, "stderr": 5.18, "cost_per_test": 0.327722, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 24.719, "latency": 667.879, "stderr": 4.599, "cost_per_test": 0.065496, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" } }, "sage": { "anthropic/claude-opus-4-7": { "accuracy": 56.103, "latency": 124.647, "stderr": 3.368, "cost_per_test": 0.407416, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "anthropic/claude-opus-4-8": { "accuracy": 54.788, "latency": 185.594, "stderr": 3.347, "cost_per_test": 0.554583, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "google/gemini-3-flash-preview": { "accuracy": 51.849, "latency": 42.697, "stderr": 3.404, "cost_per_test": 0.018229, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.5": { "accuracy": 51.532, "latency": 76.05, "stderr": 3.954, "cost_per_test": 0.14609, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 50.813, "latency": 141.066, "stderr": 3.403, "cost_per_test": 0.075919, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2.6": { "accuracy": 50.224, "latency": 270.956, "stderr": 3.429, "cost_per_test": 0.068198, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "google/gemini-3.5-flash": { "accuracy": 49.885, "latency": 32.592, "stderr": 3.438, "cost_per_test": 0.053906, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 49.54, "latency": 16.257, "stderr": 3.484, "cost_per_test": 0.0075, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3.1-pro-preview": { "accuracy": 48.677, "latency": 61.174, "stderr": 3.293, "cost_per_test": 0.068908, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-sonnet-4-6": { "accuracy": 46.582, "latency": 163.365, "stderr": 3.43, "cost_per_test": 0.234647, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "alibaba/qwen3.6-plus": { "accuracy": 44.86, "latency": 229.267, "stderr": 3.433, "cost_per_test": 0.041429, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 38.242, "latency": 66.358, "stderr": 3.425, "cost_per_test": 0.063231, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 38.081, "latency": 34.334, "stderr": 3.087, "cost_per_test": 0.007088, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-medium-3.5": { "accuracy": 37.613, "latency": 55.844, "stderr": 3.356, "cost_per_test": 0.072775, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 31.822, "latency": 66.445, "stderr": 3.134, "cost_per_test": 0.05313, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-4.3": { "accuracy": 19.736, "latency": 96.2, "stderr": 2.647, "cost_per_test": 0.037978, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" } }, "mortgage_tax": { "anthropic/claude-opus-4-7": { "accuracy": 70.27, "latency": 16.101, "stderr": 0.896, "cost_per_test": 0.084417, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "anthropic/claude-opus-4-8": { "accuracy": 69.912, "latency": 20.814, "stderr": 0.893, "cost_per_test": 0.105627, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "google/gemini-3.1-pro-preview": { "accuracy": 69.396, "latency": 23.616, "stderr": 0.908, "cost_per_test": 0.023542, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.5": { "accuracy": 68.76, "latency": 28.243, "stderr": 0.912, "cost_per_test": 0.071407, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3-flash-preview": { "accuracy": 68.72, "latency": 12.838, "stderr": 0.914, "cost_per_test": 0.007477, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3.5-flash": { "accuracy": 68.124, "latency": 11.428, "stderr": 0.91, "cost_per_test": 0.017873, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 68.044, "latency": 5.368, "stderr": 0.912, "cost_per_test": 0.00284, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3.6-plus": { "accuracy": 67.965, "latency": 78.18, "stderr": 0.908, "cost_per_test": 0.013476, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "anthropic/claude-sonnet-4-6": { "accuracy": 67.726, "latency": 29.74, "stderr": 0.921, "cost_per_test": 0.04467, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "kimi/kimi-k2.6": { "accuracy": 65.818, "latency": 109.76, "stderr": 0.93, "cost_per_test": 0.02869, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 63.514, "latency": 126.454, "stderr": 0.906, "cost_per_test": 0.044769, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 62.162, "latency": 29.792, "stderr": 0.962, "cost_per_test": 0.020742, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 59.102, "latency": 14.526, "stderr": 0.964, "cost_per_test": 0.002585, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4.3": { "accuracy": 48.252, "latency": 443.266, "stderr": 0.973, "cost_per_test": 0.012184, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 45.35, "latency": 11.124, "stderr": 0.988, "cost_per_test": 0.024144, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" } }, "vibe_code_bench": { "anthropic/claude-opus-4-8": { "accuracy": 89.869, "latency": 4549.256, "stderr": 2.244, "cost_per_test": 5.094422, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "anthropic/claude-opus-4-7": { "accuracy": 77.701, "latency": 2141.902, "stderr": 5.336, "cost_per_test": 21.407199, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "openai/gpt-5.5": { "accuracy": 77.099, "latency": 1911.567, "stderr": 7.028, "cost_per_test": 16.661911, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-6": { "accuracy": 56.223, "latency": 1572.118, "stderr": 6.546, "cost_per_test": 5.907892, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "google/gemini-3.5-flash": { "accuracy": 54.709, "latency": 893.074, "stderr": 7.002, "cost_per_test": 2.540385, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 42.523, "latency": 2055.117, "stderr": 8.901, "cost_per_test": 1.191299, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2.6": { "accuracy": 42.323, "latency": 2967.768, "stderr": 8.392, "cost_per_test": 1.925351, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "google/gemini-3.1-pro-preview": { "accuracy": 34.214, "latency": 1211.906, "stderr": 6.444, "cost_per_test": 3.825202, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 30.032, "latency": 3276.224, "stderr": 8.282, "cost_per_test": 1.277298, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3.6-plus": { "accuracy": 28.253, "latency": 2289.977, "stderr": 6.126, "cost_per_test": 5.447636, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "google/gemini-3-flash-preview": { "accuracy": 21.954, "latency": 806.66, "stderr": 6.673, "cost_per_test": 0.942067, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4.3": { "accuracy": 15.475, "latency": 589.407, "stderr": 5.576, "cost_per_test": 1.28472, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 9.882, "latency": 775.882, "stderr": 3.549, "cost_per_test": 1.306474, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 1.515, "latency": 301.651, "stderr": 1.515, "cost_per_test": 0.771885, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0, "latency": 572.711, "stderr": 0, "cost_per_test": 0.51, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" } } } }