{ "metadata": { "benchmark": "Vals Index", "slug": "vals_index_v1", "description": "Benchmark consisting of a weighted performance across finance and coding tasks. Showing the potential impact that LLM's can have on the economy.", "benchmark_id": "vals_index_v1", "family": "vals_index", "version": "1", "updated": "2026-05-06", "dataset_type": "private", "industry": "index", "tasks": { "overall": "Overall", "finance_agent": "Finance Agent v1.1", "corp_fin_v2": "CorpFin v2", "swebench": "SWE-bench Verified", "terminal_bench_2": "Terminal-Bench 2.0", "vibe_code_bench": "Vibe Code Bench" }, "models": [ "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-27b", "alibaba/qwen3.6-plus", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "deepseek/deepseek-v4-pro", "fireworks/deepseek-v3p2-thinking", "google/gemini-2.5-pro", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "meta/muse_spark", "minimax/MiniMax-M2.7", "mistralai/mistral-small-2603", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "zai/glm-5-thinking", "zai/glm-5.1" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "platform", "mode": "agentic", "archived": true, "total_models": 35 }, "tasks": { "overall": { "anthropic/claude-opus-4-7": { "accuracy": 72.208, "latency": 720.676, "stderr": 1.948, "cost_per_test": 5.507078, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic" }, "openai/gpt-5.5": { "accuracy": 71.004, "latency": 1112.559, "stderr": 2.182, "cost_per_test": 4.801356, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.4-2026-03-05": { "accuracy": 66.134, "latency": 1484.315, "stderr": 2.17, "cost_per_test": 2.787245, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 66.005, "latency": 605.296, "stderr": 2.158, "cost_per_test": 2.495345, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "anthropic/claude-sonnet-4-6": { "accuracy": 65.548, "latency": 637.775, "stderr": 2.144, "cost_per_test": 1.951158, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 64.517, "latency": 1412.048, "stderr": 2.229, "cost_per_test": 4.205442, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "deepseek/deepseek-v4-pro": { "accuracy": 62.221, "latency": 1109.719, "stderr": 2.3, "cost_per_test": 0.759748, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek" }, "kimi/kimi-k2.6": { "accuracy": 59.33, "latency": 957.903, "stderr": 2.463, "cost_per_test": 0.594522, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "google/gemini-3.1-pro-preview": { "accuracy": 58.106, "latency": 479.322, "stderr": 2.194, "cost_per_test": 1.256644, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "zai/glm-5.1": { "accuracy": 57.925, "latency": 851.096, "stderr": 2.254, "cost_per_test": 0.795339, "temperature": 1, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "meta/muse_spark": { "accuracy": 56.581, "latency": 513.403, "stderr": 2.156, "cost_per_test": 0.23782, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 55.714, "latency": 1125.058, "stderr": 2.525, "cost_per_test": 0.656206, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "zai/glm-5-thinking": { "accuracy": 55.09, "latency": 3120.378, "stderr": 2.139, "cost_per_test": 8.334239, "temperature": 1, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 54.599, "latency": 698.402, "stderr": 1.977, "cost_per_test": 7.440964, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 53.649, "latency": 774.358, "stderr": 2.232, "cost_per_test": 0.870146, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3.6-plus": { "accuracy": 53.335, "latency": 792.231, "stderr": 2.167, "cost_per_test": 1.337165, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 53.236, "latency": 864.172, "stderr": 2.081, "cost_per_test": 1.940056, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-3-flash-preview": { "accuracy": 53.161, "latency": 491.964, "stderr": 2.243, "cost_per_test": 0.359796, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4.3": { "accuracy": 52.52, "latency": 693.562, "stderr": 2.079, "cost_per_test": 0.629746, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-3-pro-preview": { "accuracy": 52.119, "latency": 2283.92, "stderr": 1.931, "cost_per_test": 1.697135, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 51.882, "latency": 1261.683, "stderr": 2.444, "cost_per_test": 0.334884, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2.5-thinking": { "accuracy": 51.697, "latency": 807.177, "stderr": 1.993, "cost_per_test": 0.28644, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 50.812, "latency": 1155.118, "stderr": 1.971, "cost_per_test": 1.066794, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "minimax/MiniMax-M2.7": { "accuracy": 49.854, "latency": 890.725, "stderr": 1.969, "cost_per_test": 0.721388, "temperature": 1, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "openai/gpt-5-2025-08-07": { "accuracy": 48.993, "latency": 859.407, "stderr": 2.038, "cost_per_test": 0.576959, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3.6-27b": { "accuracy": 48.703, "latency": 2663.466, "stderr": 1.657, "cost_per_test": 1.287196, "temperature": 1, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 47.22, "latency": 364.823, "stderr": 1.924, "cost_per_test": 0.50043, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 46.066, "latency": 138.861, "stderr": 1.808, "cost_per_test": 0.399474, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 44.474, "latency": 534.469, "stderr": 2, "cost_per_test": 0.117809, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-fast-reasoning": { "accuracy": 42.637, "latency": 416.189, "stderr": 1.772, "cost_per_test": 0.062302, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 42.501, "latency": 254.391, "stderr": 1.756, "cost_per_test": 0.074204, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 40.772, "latency": 300.754, "stderr": 1.774, "cost_per_test": 0.15555, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.5-pro": { "accuracy": 39.322, "latency": 999.357, "stderr": 1.771, "cost_per_test": 0.505033, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/mistral-small-2603": { "accuracy": 29.068, "latency": 553.591, "stderr": 1.605, "cost_per_test": 1.394531, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 26.34, "latency": 1491.735, "stderr": 1.125, "cost_per_test": 0.725441, "temperature": 1, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" } }, "finance_agent": { "anthropic/claude-opus-4-7": { "accuracy": 72.222, "latency": 273.714, "stderr": 4.748, "cost_per_test": 1.33672, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic" }, "anthropic/claude-sonnet-4-6": { "accuracy": 68.889, "latency": 347.536, "stderr": 4.907, "cost_per_test": 1.334413, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 64.444, "latency": 312.428, "stderr": 5.074, "cost_per_test": 1.119515, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "meta/muse_spark": { "accuracy": 64.444, "latency": 475.798, "stderr": 5.082, "cost_per_test": 0.064385, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta" }, "deepseek/deepseek-v4-pro": { "accuracy": 64.444, "latency": 653.19, "stderr": 5.074, "cost_per_test": 0.710285, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek" }, "zai/glm-5.1": { "accuracy": 64.444, "latency": 715.061, "stderr": 5.074, "cost_per_test": 0.317186, "temperature": 1, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 63.333, "latency": 593.047, "stderr": 5.108, "cost_per_test": 0.99647, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.5": { "accuracy": 63.333, "latency": 899.814, "stderr": 5.108, "cost_per_test": 3.732358, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 62.222, "latency": 210.976, "stderr": 5.139, "cost_per_test": 1.115022, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "zai/glm-5-thinking": { "accuracy": 61.111, "latency": 601.605, "stderr": 5.167, "cost_per_test": 0.543253, "temperature": 1, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 60, "latency": 190.087, "stderr": 5.193, "cost_per_test": 1.531755, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic" }, "openai/gpt-5.4-2026-03-05": { "accuracy": 60, "latency": 742.706, "stderr": 5.193, "cost_per_test": 1.555334, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 60, "latency": 927.737, "stderr": 5.193, "cost_per_test": 0.517677, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3-pro-preview": { "accuracy": 58.889, "latency": 150.386, "stderr": 5.216, "cost_per_test": 0.610614, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-3.1-pro-preview": { "accuracy": 58.889, "latency": 284.479, "stderr": 5.216, "cost_per_test": 0.976233, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3.6-plus": { "accuracy": 58.889, "latency": 307.156, "stderr": 5.216, "cost_per_test": 0.152637, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 58.889, "latency": 569.69, "stderr": 5.216, "cost_per_test": 0.483019, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2.5-thinking": { "accuracy": 57.778, "latency": 265.478, "stderr": 5.235, "cost_per_test": 0.174563, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 57.778, "latency": 334.677, "stderr": 5.235, "cost_per_test": 0.256637, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "kimi/kimi-k2.6": { "accuracy": 57.778, "latency": 532.653, "stderr": 5.235, "cost_per_test": 0.489502, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "grok/grok-4.3": { "accuracy": 57.778, "latency": 772.276, "stderr": 5.235, "cost_per_test": 0.427426, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 56.667, "latency": 215.762, "stderr": 5.253, "cost_per_test": 0.049297, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "minimax/MiniMax-M2.7": { "accuracy": 56.667, "latency": 1264.086, "stderr": 5.253, "cost_per_test": 0.164241, "temperature": 1, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 55.556, "latency": 98.738, "stderr": 5.267, "cost_per_test": 0.064312, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-3-flash-preview": { "accuracy": 55.556, "latency": 855.084, "stderr": 5.267, "cost_per_test": 0.343008, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3.6-27b": { "accuracy": 55, "latency": 584.833, "stderr": 3.727, "cost_per_test": 0.39447, "temperature": 1, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 54.444, "latency": 113.952, "stderr": 5.279, "cost_per_test": 0.364612, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5-2025-08-07": { "accuracy": 53.333, "latency": 1047.445, "stderr": 5.288, "cost_per_test": 0.507004, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 52.222, "latency": 70.486, "stderr": 5.295, "cost_per_test": 0.353635, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 52.222, "latency": 640.481, "stderr": 5.295, "cost_per_test": 0.150327, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-fast-reasoning": { "accuracy": 48.889, "latency": 60.939, "stderr": 5.299, "cost_per_test": 0.049119, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 48.889, "latency": 122.395, "stderr": 5.299, "cost_per_test": 0.072996, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "google/gemini-2.5-pro": { "accuracy": 43.333, "latency": 1834.194, "stderr": 5.253, "cost_per_test": 0.324011, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/mistral-small-2603": { "accuracy": 26.667, "latency": 208.54, "stderr": 4.687, "cost_per_test": 0.084983, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 3.333, "latency": 2406.288, "stderr": 1.903, "cost_per_test": 0.229225, "temperature": 1, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" } }, "corp_fin_v2": { "grok/grok-4-fast-reasoning": { "accuracy": 71.096, "latency": 10.442, "stderr": 1.548, "cost_per_test": 0.043356, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "meta/muse_spark": { "accuracy": 70.513, "latency": 39.26, "stderr": 1.557, "cost_per_test": 0.010715, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta" }, "kimi/kimi-k2.5-thinking": { "accuracy": 70.163, "latency": 69.331, "stderr": 1.562, "cost_per_test": 0.043611, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "grok/grok-4.3": { "accuracy": 70.047, "latency": 24.725, "stderr": 1.564, "cost_per_test": 0.081293, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5.5": { "accuracy": 69.58, "latency": 26.764, "stderr": 1.571, "cost_per_test": 0.43166, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 68.998, "latency": 22.376, "stderr": 1.579, "cost_per_test": 0.478497, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "kimi/kimi-k2.6": { "accuracy": 68.182, "latency": 96.974, "stderr": 1.59, "cost_per_test": 0.042064, "temperature": null, "top_p": 0.95, "max_output_tokens": 48000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 67.832, "latency": 40.886, "stderr": 1.595, "cost_per_test": 0.034482, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "google/gemini-3-flash-preview": { "accuracy": 67.716, "latency": 14.833, "stderr": 1.596, "cost_per_test": 0.039017, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 67.599, "latency": 18.561, "stderr": 1.598, "cost_per_test": 0.012735, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "zai/glm-5-thinking": { "accuracy": 67.016, "latency": 70.517, "stderr": 1.605, "cost_per_test": 0.073316, "temperature": 1, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "openai/gpt-5.4-2026-03-05": { "accuracy": 66.783, "latency": 122.651, "stderr": 1.608, "cost_per_test": 0.235266, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-sonnet-4-6": { "accuracy": 66.667, "latency": 20.037, "stderr": 1.609, "cost_per_test": 0.287644, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 66.2, "latency": 20.361, "stderr": 1.615, "cost_per_test": 0.475505, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic" }, "anthropic/claude-opus-4-7": { "accuracy": 66.084, "latency": 18.221, "stderr": 1.616, "cost_per_test": 0.816017, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 65.851, "latency": 9.418, "stderr": 1.619, "cost_per_test": 0.130871, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-3.1-pro-preview": { "accuracy": 65.734, "latency": 28.137, "stderr": 1.62, "cost_per_test": 0.145587, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 65.734, "latency": 31.057, "stderr": 1.62, "cost_per_test": 0.109551, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "zai/glm-5.1": { "accuracy": 65.268, "latency": 45.736, "stderr": 1.625, "cost_per_test": 0.071771, "temperature": 1, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 65.035, "latency": 32.888, "stderr": 1.628, "cost_per_test": 0.090475, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3-pro-preview": { "accuracy": 64.569, "latency": 23.778, "stderr": 1.633, "cost_per_test": 0.154515, "temperature": 1, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 64.219, "latency": 21.997, "stderr": 1.636, "cost_per_test": 0.286695, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "minimax/MiniMax-M2.7": { "accuracy": 63.753, "latency": 11.623, "stderr": 1.641, "cost_per_test": 0.023635, "temperature": 1, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 63.17, "latency": 15.049, "stderr": 1.647, "cost_per_test": 0.096327, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "google/gemini-2.5-pro": { "accuracy": 62.587, "latency": 28.746, "stderr": 1.652, "cost_per_test": 0.081072, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "alibaba/qwen3.6-27b": { "accuracy": 62.354, "latency": 51.777, "stderr": 1.654, "cost_per_test": 0.046263, "temperature": 1, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5-2025-08-07": { "accuracy": 62.354, "latency": 158.5, "stderr": 1.654, "cost_per_test": 0.079354, "temperature": null, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3.6-plus": { "accuracy": 62.238, "latency": 35.919, "stderr": 1.655, "cost_per_test": 0.038254, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "deepseek/deepseek-v4-pro": { "accuracy": 61.888, "latency": 74.027, "stderr": 1.658, "cost_per_test": 0.148855, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 61.189, "latency": 109.941, "stderr": 1.664, "cost_per_test": 0.019795, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 61.072, "latency": 7.363, "stderr": 1.665, "cost_per_test": 0.01612, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 60.839, "latency": 4.708, "stderr": 1.666, "cost_per_test": 0.022927, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 60.256, "latency": 45.279, "stderr": 1.671, "cost_per_test": 0.068348, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "mistralai/mistral-small-2603": { "accuracy": 51.166, "latency": 14.22, "stderr": 1.707, "cost_per_test": 0.011286, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 49.301, "latency": 109.73, "stderr": 1.707, "cost_per_test": 0.037839, "temperature": 1, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" } }, "swebench": { "openai/gpt-5.5": { "accuracy": 82.6, "latency": 426.425, "stderr": 1.695, "cost_per_test": 1.361765, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-opus-4-7": { "accuracy": 82.353, "latency": 508.652, "stderr": 3.775, "cost_per_test": 2.999546, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 77.451, "latency": 633.682, "stderr": 4.138, "cost_per_test": 1.706873, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 75.49, "latency": 353.853, "stderr": 4.259, "cost_per_test": 0.94669, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "deepseek/deepseek-v4-pro": { "accuracy": 75.49, "latency": 574.117, "stderr": 4.259, "cost_per_test": 0.436763, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "kimi/kimi-k2.6": { "accuracy": 74.51, "latency": 396.661, "stderr": 4.315, "cost_per_test": 0.322908, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "google/gemini-3.1-pro-preview": { "accuracy": 72.549, "latency": 329.202, "stderr": 4.419, "cost_per_test": 0.820116, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 72.549, "latency": 407.124, "stderr": 4.419, "cost_per_test": 1.459323, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "google/gemini-3-pro-preview": { "accuracy": 72.549, "latency": 430.643, "stderr": 4.419, "cost_per_test": 0.847172, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "zai/glm-5.1": { "accuracy": 72.549, "latency": 598.277, "stderr": 4.419, "cost_per_test": 0.534539, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 71.569, "latency": 336.507, "stderr": 4.466, "cost_per_test": 1.202144, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 71.569, "latency": 722.109, "stderr": 4.466, "cost_per_test": 1.706049, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "grok/grok-4.3": { "accuracy": 70.588, "latency": 126.579, "stderr": 4.512, "cost_per_test": 0.23972, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "alibaba/qwen3.6-plus": { "accuracy": 70.588, "latency": 487.435, "stderr": 4.512, "cost_per_test": 0.901327, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "alibaba/qwen3.6-27b": { "accuracy": 70.588, "latency": 910.234, "stderr": 4.512, "cost_per_test": 1.041977, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "zai/glm-5-thinking": { "accuracy": 69.608, "latency": 598.373, "stderr": 4.554, "cost_per_test": 0.537135, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "minimax/MiniMax-M2.7": { "accuracy": 69.608, "latency": 961.544, "stderr": 4.554, "cost_per_test": 0.525961, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "google/gemini-3-flash-preview": { "accuracy": 68.627, "latency": 327.172, "stderr": 4.594, "cost_per_test": 0.306186, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 67.647, "latency": 281.133, "stderr": 4.632, "cost_per_test": 0.107316, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 67.647, "latency": 355.599, "stderr": 4.632, "cost_per_test": 0.589598, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 67.647, "latency": 388.838, "stderr": 4.632, "cost_per_test": 0.890462, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "meta/muse_spark": { "accuracy": 67.647, "latency": 504.333, "stderr": 4.632, "cost_per_test": 0.464053, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 66.667, "latency": 151.343, "stderr": 4.668, "cost_per_test": 0.586871, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 66.667, "latency": 374.288, "stderr": 4.668, "cost_per_test": 0.194738, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-5-2025-08-07": { "accuracy": 65.686, "latency": 252.837, "stderr": 4.701, "cost_per_test": 0.296208, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 65.686, "latency": 507.258, "stderr": 4.701, "cost_per_test": 0.917096, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 65.686, "latency": 546.856, "stderr": 4.701, "cost_per_test": 0.794681, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 64.706, "latency": 264.186, "stderr": 4.732, "cost_per_test": 0.407017, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 63.725, "latency": 551.963, "stderr": 4.761, "cost_per_test": 0.724456, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 57.843, "latency": 136.077, "stderr": 4.889, "cost_per_test": 0.106331, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 55.882, "latency": 195.055, "stderr": 4.916, "cost_per_test": 0.053604, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "google/gemini-2.5-pro": { "accuracy": 49.02, "latency": 240.174, "stderr": 4.95, "cost_per_test": 0.424246, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "mistralai/mistral-small-2603": { "accuracy": 43.137, "latency": 322.648, "stderr": 4.904, "cost_per_test": 0.297624, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "grok/grok-4-fast-reasoning": { "accuracy": 42.157, "latency": 129.441, "stderr": 4.889, "cost_per_test": 0.051519, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 36.275, "latency": 147.938, "stderr": 4.761, "cost_per_test": 0.029005, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": null } }, "terminal_bench_2": { "openai/gpt-5.5": { "accuracy": 73.202, "latency": 2298.224, "stderr": 3.978, "cost_per_test": 1.819086, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-7": { "accuracy": 68.539, "latency": 660.89, "stderr": 4.95, "cost_per_test": 0.975907, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic" }, "google/gemini-3.1-pro-preview": { "accuracy": 67.416, "latency": 542.887, "stderr": 4.996, "cost_per_test": 0.516084, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "meta/muse_spark": { "accuracy": 59.551, "latency": 566.325, "stderr": 5.232, "cost_per_test": 0.070242, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta" }, "anthropic/claude-sonnet-4-6": { "accuracy": 59.551, "latency": 615.5, "stderr": 5.232, "cost_per_test": 0.518966, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 58.427, "latency": 898.494, "stderr": 5.254, "cost_per_test": 1.140327, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "openai/gpt-5.4-2026-03-05": { "accuracy": 58.427, "latency": 1027.517, "stderr": 5.254, "cost_per_test": 0.511959, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2.6": { "accuracy": 57.303, "latency": 795.458, "stderr": 5.273, "cost_per_test": 0.192786, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "deepseek/deepseek-v4-pro": { "accuracy": 56.18, "latency": 828.342, "stderr": 5.289, "cost_per_test": 0.296847, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": false, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "DeepSeek" }, "google/gemini-3-pro-preview": { "accuracy": 55.056, "latency": 416.249, "stderr": 5.303, "cost_per_test": 0.455782, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 53.933, "latency": 661.913, "stderr": 5.314, "cost_per_test": 1.122133, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic" }, "zai/glm-5.1": { "accuracy": 53.933, "latency": 881.675, "stderr": 5.314, "cost_per_test": 0.158286, "temperature": 1, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "google/gemini-3-flash-preview": { "accuracy": 51.685, "latency": 456.071, "stderr": 5.327, "cost_per_test": 0.168702, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 51.685, "latency": 742.684, "stderr": 5.327, "cost_per_test": 0.469958, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "zai/glm-5-thinking": { "accuracy": 49.438, "latency": 875.602, "stderr": 5.33, "cost_per_test": 0.247234, "temperature": 0.7, "top_p": 1, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "minimax/MiniMax-M2.7": { "accuracy": 47.191, "latency": 839.048, "stderr": 5.322, "cost_per_test": 0.073995, "temperature": 1, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "alibaba/qwen3.6-plus": { "accuracy": 44.944, "latency": 840.666, "stderr": 5.303, "cost_per_test": 0.14597, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 44.944, "latency": 886.169, "stderr": 5.303, "cost_per_test": 0.407628, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3.6-27b": { "accuracy": 44.944, "latency": 2007.631, "stderr": 5.303, "cost_per_test": 0.298145, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 44.944, "latency": 2241.556, "stderr": 4.492, "cost_per_test": 0.914107, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4.3": { "accuracy": 43.446, "latency": 1954.824, "stderr": 4.406, "cost_per_test": 1.115571, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 41.573, "latency": 736.918, "stderr": 5.254, "cost_per_test": 0.751119, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 41.573, "latency": 1877.151, "stderr": 5.254, "cost_per_test": 0.321994, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 40.449, "latency": 161.408, "stderr": 5.232, "cost_per_test": 0.154109, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "kimi/kimi-k2.5-thinking": { "accuracy": 40.449, "latency": 756.405, "stderr": 5.232, "cost_per_test": 0.139997, "temperature": null, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 39.888, "latency": 2527.932, "stderr": 4.454, "cost_per_test": 0.224388, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 38.202, "latency": 655.044, "stderr": 5.18, "cost_per_test": 0.327722, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "openai/gpt-5-2025-08-07": { "accuracy": 37.079, "latency": 986.102, "stderr": 5.149, "cost_per_test": 0.477064, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 35.955, "latency": 1024.966, "stderr": 5.115, "cost_per_test": 0.166362, "temperature": 1, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "google/gemini-2.5-pro": { "accuracy": 30.337, "latency": 796.526, "stderr": 4.901, "cost_per_test": 0.472003, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4-fast-reasoning": { "accuracy": 29.213, "latency": 431.268, "stderr": 4.848, "cost_per_test": 0.03431, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 26.966, "latency": 1028.621, "stderr": 4.731, "cost_per_test": 0.116731, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 24.719, "latency": 479.157, "stderr": 4.599, "cost_per_test": 0.055526, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 24.719, "latency": 667.879, "stderr": 4.599, "cost_per_test": 0.065496, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "mistralai/mistral-small-2603": { "accuracy": 16.854, "latency": 407.865, "stderr": 3.991, "cost_per_test": 0.105762, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI" } }, "vibe_code_bench": { "anthropic/claude-opus-4-7": { "accuracy": 77.701, "latency": 2141.902, "stderr": 5.336, "cost_per_test": 21.407199, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "openai/gpt-5.5": { "accuracy": 77.099, "latency": 1911.567, "stderr": 7.028, "cost_per_test": 16.661911, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.4-2026-03-05": { "accuracy": 73.143, "latency": 5174.848, "stderr": 6.271, "cost_per_test": 10.686976, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "openai/gpt-5.2-2025-12-11": { "accuracy": 67.361, "latency": 4971.341, "stderr": 6.821, "cost_per_test": 17.745182, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 64.478, "latency": 1386.056, "stderr": 6.364, "cost_per_test": 8.279062, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "anthropic/claude-sonnet-4-6": { "accuracy": 56.223, "latency": 1572.118, "stderr": 6.546, "cost_per_test": 5.907892, "temperature": 1, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic" }, "deepseek/deepseek-v4-pro": { "accuracy": 55.905, "latency": 3418.92, "stderr": 7.421, "cost_per_test": 2.205991, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek" }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 42.523, "latency": 2055.117, "stderr": 8.901, "cost_per_test": 1.191299, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "kimi/kimi-k2.6": { "accuracy": 42.323, "latency": 2967.768, "stderr": 8.392, "cost_per_test": 1.925351, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "google/gemini-3.1-pro-preview": { "accuracy": 34.214, "latency": 1211.906, "stderr": 6.444, "cost_per_test": 3.825202, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "zai/glm-5.1": { "accuracy": 32.804, "latency": 2014.732, "stderr": 7.069, "cost_per_test": 2.894911, "temperature": 1, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 30.032, "latency": 3276.224, "stderr": 8.282, "cost_per_test": 1.277298, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "alibaba/qwen3.6-plus": { "accuracy": 28.253, "latency": 2289.977, "stderr": 6.126, "cost_per_test": 5.447636, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "openai/gpt-5.1-2025-11-13": { "accuracy": 28.231, "latency": 1836.188, "stderr": 6.633, "cost_per_test": 2.574926, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "zai/glm-5-thinking": { "accuracy": 25.018, "latency": 13455.794, "stderr": 5.996, "cost_per_test": 40.270258, "temperature": 1, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI" }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 23.337, "latency": 2962.133, "stderr": 5.528, "cost_per_test": 6.656984, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 22.156, "latency": 2283.142, "stderr": 4.45, "cost_per_test": 32.873283, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic" }, "google/gemini-3-flash-preview": { "accuracy": 21.954, "latency": 806.66, "stderr": 6.673, "cost_per_test": 0.942067, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5-2025-08-07": { "accuracy": 21.314, "latency": 1852.149, "stderr": 4.816, "cost_per_test": 1.525167, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "meta/muse_spark": { "accuracy": 18.429, "latency": 981.298, "stderr": 6.31, "cost_per_test": 0.579705, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta" }, "grok/grok-4.3": { "accuracy": 15.475, "latency": 589.407, "stderr": 5.576, "cost_per_test": 1.28472, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "kimi/kimi-k2.5-thinking": { "accuracy": 14.769, "latency": 2570.381, "stderr": 4.525, "cost_per_test": 0.879292, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI" }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 13.731, "latency": 3015.618, "stderr": 4.258, "cost_per_test": 3.803761, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "google/gemini-3-pro-preview": { "accuracy": 12.978, "latency": 10398.545, "stderr": 3.879, "cost_per_test": 6.417592, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 12.576, "latency": 698.246, "stderr": 4.46, "cost_per_test": 0.248587, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI" }, "minimax/MiniMax-M2.7": { "accuracy": 11.719, "latency": 1377.326, "stderr": 4.172, "cost_per_test": 2.819109, "temperature": 1, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax" }, "alibaba/qwen3.6-27b": { "accuracy": 11.142, "latency": 9762.856, "stderr": 4.329, "cost_per_test": 4.655127, "temperature": 1, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba" }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 9.882, "latency": 775.882, "stderr": 3.549, "cost_per_test": 1.306474, "temperature": 1, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic" }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 2.904, "latency": 3365.726, "stderr": 2.072, "cost_per_test": 2.469325, "temperature": 1, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI" }, "grok/grok-4.20-0309-reasoning": { "accuracy": 1.515, "latency": 301.651, "stderr": 1.515, "cost_per_test": 0.771885, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "grok/grok-4-1-fast-reasoning": { "accuracy": 0, "latency": 527.562, "stderr": 0, "cost_per_test": 0.209444, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0, "latency": 572.711, "stderr": 0, "cost_per_test": 0.51, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google" }, "grok/grok-4-fast-reasoning": { "accuracy": 0, "latency": 1448.855, "stderr": 0, "cost_per_test": 0.133206, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI" }, "mistralai/mistral-small-2603": { "accuracy": 0, "latency": 1814.681, "stderr": 0, "cost_per_test": 6.473001, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI" }, "google/gemini-2.5-pro": { "accuracy": 0, "latency": 2097.144, "stderr": 0, "cost_per_test": 1.223831, "temperature": 1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google" } } } }