{ "metadata": { "benchmark": "Vals Index", "slug": "vals_index_v1_1", "description": "Benchmark consisting of a weighted performance across finance and coding tasks. Showing the potential impact that LLM's can have on the economy.", "benchmark_id": "vals_index_v1_1", "family": "vals_index", "version": "1.1", "updated": "2026-05-28", "dataset_type": "private", "industry": "index", "tasks": { "overall": "Overall", "finance_agent": "Finance Agent v2", "corp_fin_v2": "CorpFin v2", "swebench": "SWE-bench", "terminal_bench_2": "Terminal-Bench 2.0", "vibe_code_bench": "Vibe Code Bench" }, "models": [ "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-sonnet-4-6", "deepseek/deepseek-v4-pro", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "kimi/kimi-k2.6", "minimax/MiniMax-M2.7", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "zai/glm-5.1" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "platform", "mode": "agentic", "archived": true, "total_models": 19 }, "tasks": { "overall": { "anthropic/claude-opus-4-8": { "accuracy": 70.166, "latency": 1239.013, "stderr": 0.91, "cost_per_test": 2.595116, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "openai/gpt-5.5": { "accuracy": 67.622, "latency": 1055.908, "stderr": 1.589, "cost_per_test": 4.811797, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-7": { "accuracy": 66.099, "latency": 730.388, "stderr": 1.36, "cost_per_test": 5.967337, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic" }, "google/gemini-3.5-flash": { "accuracy": 62.054, "latency": 371.334, "stderr": 1.608, "cost_per_test": 1.466409, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-sonnet-4-6": { "accuracy": 60.296, "latency": 694.087, "stderr": 1.616, "cost_per_test": 2.141135, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "alibaba/qwen3.7-max": { "accuracy": 57.294, "latency": 1777.638, "stderr": 1.578, "cost_per_test": 5.300715, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "deepseek/deepseek-v4-pro": { "accuracy": 56.231, "latency": 1269.089, "stderr": 1.764, "cost_per_test": 0.837724, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek" }, "kimi/kimi-k2.6": { "accuracy": 55.551, "latency": 1157.017, "stderr": 1.987, "cost_per_test": 0.713166, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "google/gemini-3.1-pro-preview": { "accuracy": 53.423, "latency": 461.64, "stderr": 1.668, "cost_per_test": 1.33708, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "zai/glm-5.1": { "accuracy": 52.144, "latency": 887.644, "stderr": 1.692, "cost_per_test": 0.848758, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 51.422, "latency": 1323.282, "stderr": 2.052, "cost_per_test": 0.760107, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3-flash-preview": { "accuracy": 49.314, "latency": 369.269, "stderr": 1.634, "cost_per_test": 0.399929, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3.6-plus": { "accuracy": 48.039, "latency": 847.84, "stderr": 1.545, "cost_per_test": 1.432742, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "grok/grok-4.3": { "accuracy": 46.635, "latency": 663.58, "stderr": 1.412, "cost_per_test": 0.712693, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 46.461, "latency": 1283.782, "stderr": 1.897, "cost_per_test": 0.355489, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "minimax/MiniMax-M2.7": { "accuracy": 41.406, "latency": 788.051, "stderr": 1.26, "cost_per_test": 0.722411, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 40.325, "latency": 378.043, "stderr": 1.136, "cost_per_test": 0.550923, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 39.11, "latency": 195.123, "stderr": 0.936, "cost_per_test": 0.489729, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 35.236, "latency": 286.99, "stderr": 0.874, "cost_per_test": 0.167645, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" } }, "finance_agent": { "google/gemini-3.5-flash": { "accuracy": 58.101, "latency": 316.567, "stderr": 0.874, "cost_per_test": 2.505488, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-8": { "accuracy": 53.949, "latency": 522.678, "stderr": 1.764, "cost_per_test": 4.218538, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "openai/gpt-5.5": { "accuracy": 51.834, "latency": 616.56, "stderr": 0.461, "cost_per_test": 3.784562, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-7": { "accuracy": 51.45, "latency": 322.275, "stderr": 0.251, "cost_per_test": 3.638017, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic" }, "anthropic/claude-sonnet-4-6": { "accuracy": 51.033, "latency": 629.1, "stderr": 1.076, "cost_per_test": 2.284299, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "alibaba/qwen3.7-max": { "accuracy": 48.353, "latency": 483.912, "stderr": 0.477, "cost_per_test": 1.971421, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 45.407, "latency": 1918.861, "stderr": 1.406, "cost_per_test": 1.037185, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2.6": { "accuracy": 44.932, "latency": 1528.223, "stderr": 1.709, "cost_per_test": 1.082722, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "zai/glm-5.1": { "accuracy": 44.79, "latency": 897.798, "stderr": 0.347, "cost_per_test": 0.584284, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "deepseek/deepseek-v4-pro": { "accuracy": 44.081, "latency": 1450.039, "stderr": 0.767, "cost_per_test": 1.100162, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek" }, "google/gemini-3.1-pro-preview": { "accuracy": 42.968, "latency": 196.068, "stderr": 1.931, "cost_per_test": 1.378409, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3-flash-preview": { "accuracy": 42.476, "latency": 241.608, "stderr": 0.666, "cost_per_test": 0.543671, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3.6-plus": { "accuracy": 40.88, "latency": 585.203, "stderr": 0.729, "cost_per_test": 0.630525, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 38.236, "latency": 326.257, "stderr": 0.381, "cost_per_test": 0.152321, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4.3": { "accuracy": 37.766, "latency": 622.363, "stderr": 0.698, "cost_per_test": 0.842163, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 30.999, "latency": 180.056, "stderr": 0.088, "cost_per_test": 0.617074, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 30.068, "latency": 53.577, "stderr": 0.727, "cost_per_test": 0.133469, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 28.573, "latency": 351.797, "stderr": 0.598, "cost_per_test": 0.804911, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "minimax/MiniMax-M2.7": { "accuracy": 27.943, "latency": 750.712, "stderr": 1.058, "cost_per_test": 0.169353, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" } }, "corp_fin_v2": { "grok/grok-4.3": { "accuracy": 70.047, "latency": 24.725, "stderr": 1.564, "cost_per_test": 0.081293, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5.5": { "accuracy": 69.58, "latency": 26.764, "stderr": 1.571, "cost_per_test": 0.43166, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2.6": { "accuracy": 68.182, "latency": 96.974, "stderr": 1.59, "cost_per_test": 0.042064, "temperature": null, "top_p": 0.95, "max_output_tokens": 48000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "google/gemini-3-flash-preview": { "accuracy": 67.716, "latency": 14.833, "stderr": 1.596, "cost_per_test": 0.039017, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-sonnet-4-6": { "accuracy": 66.667, "latency": 20.037, "stderr": 1.609, "cost_per_test": 0.287644, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "anthropic/claude-opus-4-7": { "accuracy": 66.084, "latency": 18.221, "stderr": 1.616, "cost_per_test": 0.816017, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "anthropic/claude-opus-4-8": { "accuracy": 65.967, "latency": 20.38, "stderr": 1.618, "cost_per_test": 0.855534, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 65.851, "latency": 9.418, "stderr": 1.619, "cost_per_test": 0.130871, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-3.1-pro-preview": { "accuracy": 65.734, "latency": 28.137, "stderr": 1.62, "cost_per_test": 0.145587, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3.7-max": { "accuracy": 65.385, "latency": 18.336, "stderr": 1.624, "cost_per_test": 0.17462, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "zai/glm-5.1": { "accuracy": 65.268, "latency": 45.736, "stderr": 1.625, "cost_per_test": 0.071771, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "google/gemini-3.5-flash": { "accuracy": 64.569, "latency": 8.593, "stderr": 1.633, "cost_per_test": 0.140208, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "minimax/MiniMax-M2.7": { "accuracy": 63.753, "latency": 11.623, "stderr": 1.641, "cost_per_test": 0.023635, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 63.17, "latency": 15.049, "stderr": 1.647, "cost_per_test": 0.096327, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "alibaba/qwen3.6-plus": { "accuracy": 62.238, "latency": 35.919, "stderr": 1.655, "cost_per_test": 0.038254, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "deepseek/deepseek-v4-pro": { "accuracy": 61.888, "latency": 74.027, "stderr": 1.658, "cost_per_test": 0.148855, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 61.072, "latency": 7.363, "stderr": 1.665, "cost_per_test": 0.01612, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 60.839, "latency": 4.708, "stderr": 1.666, "cost_per_test": 0.022927, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 60.256, "latency": 45.279, "stderr": 1.671, "cost_per_test": 0.068348, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" } }, "swebench": { "anthropic/claude-opus-4-8": { "accuracy": 89.216, "latency": 630.305, "stderr": 3.071, "cost_per_test": 1.923224, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-5.5": { "accuracy": 82.6, "latency": 426.425, "stderr": 1.695, "cost_per_test": 1.361765, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-opus-4-7": { "accuracy": 82.353, "latency": 508.652, "stderr": 3.775, "cost_per_test": 2.999546, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 77.451, "latency": 633.682, "stderr": 4.138, "cost_per_test": 1.706873, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "google/gemini-3.5-flash": { "accuracy": 75.49, "latency": 230.39, "stderr": 4.259, "cost_per_test": 0.953882, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "deepseek/deepseek-v4-pro": { "accuracy": 75.49, "latency": 574.117, "stderr": 4.259, "cost_per_test": 0.436763, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "kimi/kimi-k2.6": { "accuracy": 74.51, "latency": 396.661, "stderr": 4.315, "cost_per_test": 0.322908, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "google/gemini-3.1-pro-preview": { "accuracy": 72.549, "latency": 329.202, "stderr": 4.419, "cost_per_test": 0.820116, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "zai/glm-5.1": { "accuracy": 72.549, "latency": 598.277, "stderr": 4.419, "cost_per_test": 0.534539, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "grok/grok-4.3": { "accuracy": 70.588, "latency": 126.579, "stderr": 4.512, "cost_per_test": 0.23972, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "alibaba/qwen3.6-plus": { "accuracy": 70.588, "latency": 487.435, "stderr": 4.512, "cost_per_test": 0.901327, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "minimax/MiniMax-M2.7": { "accuracy": 69.608, "latency": 961.544, "stderr": 4.554, "cost_per_test": 0.525961, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "google/gemini-3-flash-preview": { "accuracy": 68.627, "latency": 327.172, "stderr": 4.594, "cost_per_test": 0.306186, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 67.647, "latency": 281.133, "stderr": 4.632, "cost_per_test": 0.107316, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 67.647, "latency": 355.599, "stderr": 4.632, "cost_per_test": 0.589598, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 66.667, "latency": 151.343, "stderr": 4.668, "cost_per_test": 0.586871, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "alibaba/qwen3.7-max": { "accuracy": 66.667, "latency": 1037.61, "stderr": 4.668, "cost_per_test": 3.384593, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 64.706, "latency": 264.186, "stderr": 4.732, "cost_per_test": 0.407017, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 57.843, "latency": 136.077, "stderr": 4.889, "cost_per_test": 0.106331, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null } }, "terminal_bench_2": { "openai/gpt-5.5": { "accuracy": 73.202, "latency": 2298.224, "stderr": 3.978, "cost_per_test": 1.819086, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-8": { "accuracy": 70.037, "latency": 472.445, "stderr": 1.35, "cost_per_test": 0.883864, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "anthropic/claude-opus-4-7": { "accuracy": 68.539, "latency": 660.89, "stderr": 4.95, "cost_per_test": 0.975907, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic" }, "google/gemini-3.5-flash": { "accuracy": 67.416, "latency": 408.048, "stderr": 1.297, "cost_per_test": 1.192081, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3.1-pro-preview": { "accuracy": 67.416, "latency": 542.887, "stderr": 4.996, "cost_per_test": 0.516084, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-sonnet-4-6": { "accuracy": 59.551, "latency": 615.5, "stderr": 5.232, "cost_per_test": 0.518966, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "alibaba/qwen3.7-max": { "accuracy": 59.176, "latency": 664.843, "stderr": 1.35, "cost_per_test": 0.719164, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "kimi/kimi-k2.6": { "accuracy": 57.303, "latency": 795.458, "stderr": 5.273, "cost_per_test": 0.192786, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "deepseek/deepseek-v4-pro": { "accuracy": 56.18, "latency": 828.342, "stderr": 5.289, "cost_per_test": 0.296847, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek" }, "zai/glm-5.1": { "accuracy": 53.933, "latency": 881.675, "stderr": 5.314, "cost_per_test": 0.158286, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "google/gemini-3-flash-preview": { "accuracy": 51.685, "latency": 456.071, "stderr": 5.327, "cost_per_test": 0.168702, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "minimax/MiniMax-M2.7": { "accuracy": 47.191, "latency": 839.048, "stderr": 5.322, "cost_per_test": 0.073995, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "alibaba/qwen3.6-plus": { "accuracy": 44.944, "latency": 840.666, "stderr": 5.303, "cost_per_test": 0.14597, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 44.944, "latency": 2241.556, "stderr": 4.492, "cost_per_test": 0.914107, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4.3": { "accuracy": 43.446, "latency": 1954.824, "stderr": 4.406, "cost_per_test": 1.115571, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 40.449, "latency": 161.408, "stderr": 5.232, "cost_per_test": 0.154109, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 39.888, "latency": 2527.932, "stderr": 4.454, "cost_per_test": 0.224388, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 38.202, "latency": 655.044, "stderr": 5.18, "cost_per_test": 0.327722, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 24.719, "latency": 667.879, "stderr": 4.599, "cost_per_test": 0.065496, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" } }, "vibe_code_bench": { "anthropic/claude-opus-4-8": { "accuracy": 89.869, "latency": 4549.256, "stderr": 2.244, "cost_per_test": 5.094422, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "anthropic/claude-opus-4-7": { "accuracy": 77.701, "latency": 2141.902, "stderr": 5.336, "cost_per_test": 21.407199, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "openai/gpt-5.5": { "accuracy": 77.099, "latency": 1911.567, "stderr": 7.028, "cost_per_test": 16.661911, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-6": { "accuracy": 56.223, "latency": 1572.118, "stderr": 6.546, "cost_per_test": 5.907892, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "deepseek/deepseek-v4-pro": { "accuracy": 55.905, "latency": 3418.92, "stderr": 7.421, "cost_per_test": 2.205991, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek" }, "google/gemini-3.5-flash": { "accuracy": 54.709, "latency": 893.074, "stderr": 7.002, "cost_per_test": 2.540385, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3.7-max": { "accuracy": 52.88, "latency": 6683.491, "stderr": 6.854, "cost_per_test": 20.253775, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 42.523, "latency": 2055.117, "stderr": 8.901, "cost_per_test": 1.191299, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2.6": { "accuracy": 42.323, "latency": 2967.768, "stderr": 8.392, "cost_per_test": 1.925351, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "google/gemini-3.1-pro-preview": { "accuracy": 34.214, "latency": 1211.906, "stderr": 6.444, "cost_per_test": 3.825202, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "zai/glm-5.1": { "accuracy": 32.804, "latency": 2014.732, "stderr": 7.069, "cost_per_test": 2.894911, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 30.032, "latency": 3276.224, "stderr": 8.282, "cost_per_test": 1.277298, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3.6-plus": { "accuracy": 28.253, "latency": 2289.977, "stderr": 6.126, "cost_per_test": 5.447636, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "google/gemini-3-flash-preview": { "accuracy": 21.954, "latency": 806.66, "stderr": 6.673, "cost_per_test": 0.942067, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4.3": { "accuracy": 15.475, "latency": 589.407, "stderr": 5.576, "cost_per_test": 1.28472, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "minimax/MiniMax-M2.7": { "accuracy": 11.719, "latency": 1377.326, "stderr": 4.172, "cost_per_test": 2.819109, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 9.882, "latency": 775.882, "stderr": 3.549, "cost_per_test": 1.306474, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 1.515, "latency": 301.651, "stderr": 1.515, "cost_per_test": 0.771885, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 572.711, "stderr": 0.0, "cost_per_test": 0.51, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" } } } }