{ "metadata": { "benchmark": "MMMU Pro", "slug": "mmmu", "description": "Multimodal Multi-task Benchmark", "benchmark_id": "mmmu", "family": "mmmu", "version": "1", "updated": "2026-09-01", "dataset_type": "public", "industry": "academic", "tasks": { "overall": "Overall" }, "models": [ "alibaba/qwen3.5-flash", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-plus", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "anthropic/claude-3-5-sonnet-20241022", "anthropic/claude-3-7-sonnet-20250219", "anthropic/claude-3-7-sonnet-20250219-thinking", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-1-20250805-thinking", "anthropic/claude-opus-4-20250514", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "fireworks/llama4-maverick-instruct-basic", "google/gemini-1.5-flash-002", "google/gemini-1.5-pro-002", "google/gemini-2.0-flash-001", "google/gemini-2.5-flash-lite-preview-09-2025", "google/gemini-2.5-flash-lite-preview-09-2025-thinking", "google/gemini-2.5-flash-preview-04-17", "google/gemini-2.5-flash-preview-04-17-thinking", "google/gemini-2.5-flash-preview-09-2025", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-pro-exp-03-25", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "grok/grok-2-vision-1212", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse_spark", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "minimax/MiniMax-M3", "mistralai/magistral-medium-2509", "mistralai/magistral-small-2509", "mistralai/mistral-large-2512", "mistralai/mistral-medium-2505", "mistralai/mistral-small-2503", "openai/gpt-4.1-2025-04-14", "openai/gpt-4.1-mini-2025-04-14", "openai/gpt-4.1-nano-2025-04-14", "openai/gpt-4o-2024-08-06", "openai/gpt-4o-2024-11-20", "openai/gpt-4o-mini-2024-07-18", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-nano-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/o1-2024-12-17", "openai/o3-2025-04-16", "openai/o4-mini-2025-04-16", "thinkingmachines/inkling", "together/meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo", "together/meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo", "together/meta-llama/Llama-4-Scout-17B-16E-Instruct", "xiaomi/mimo-v2.5", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "custom", "mode": "one-shot", "archived": false, "partner": false, "total_models": 93 }, "tasks": { "overall": { "anthropic/claude-fable-5-1": { "accuracy": 90.636, "latency": 37.47, "stderr": 0.7, "cost_per_test": 0.166285, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 89.884, "latency": 30.028, "stderr": 0.725, "cost_per_test": 0.069014, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 89.306, "latency": 61.44, "stderr": 0.743, "cost_per_test": 0.16064, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 89.075, "latency": 12.028, "stderr": 0.75, "cost_per_test": 0.004472, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 88.96, "latency": 6.283, "stderr": 0.753, "cost_per_test": 0.014052, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 88.844, "latency": 26.198, "stderr": 0.757, "cost_per_test": 0.055795, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 88.382, "latency": 11.313, "stderr": 0.77, "cost_per_test": 0.0186, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 88.266, "latency": 12.226, "stderr": 0.774, "cost_per_test": 0.025155, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 88.266, "latency": 54.147, "stderr": 0.774, "cost_per_test": 0.043319, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 88.208, "latency": 76.99, "stderr": 0.775, "cost_per_test": 0.064084, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 88.15, "latency": 106.516, "stderr": 0.777, "cost_per_test": 0.059115, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 88.035, "latency": 43.853, "stderr": 0.78, "cost_per_test": 0.015881, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 87.63, "latency": 27.856, "stderr": 0.792, "cost_per_test": 0.014422, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 87.514, "latency": 48.102, "stderr": 0.795, "cost_per_test": 0.010159, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 87.514, "latency": 65.565, "stderr": 0.795, "cost_per_test": 0.047083, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 87.399, "latency": 48.272, "stderr": 0.798, "cost_per_test": 0.001746, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 86.667, "latency": 96.324, "stderr": 0.822, "cost_per_test": 0.061121, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 86.59, "latency": 31.715, "stderr": 0.819, "cost_per_test": 0.01361, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 86.59, "latency": 49.044, "stderr": 0.819, "cost_per_test": 0.120663, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 86.474, "latency": 13.033, "stderr": 0.822, "cost_per_test": 0.016375, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 86.301, "latency": 241.908, "stderr": 0.827, "cost_per_test": 0.034867, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 86.127, "latency": 73.839, "stderr": 0.831, "cost_per_test": 0.018022, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 86.012, "latency": 28.218, "stderr": 0.834, "cost_per_test": 0.000962, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 85.549, "latency": 70.801, "stderr": 0.845, "cost_per_test": 0.170461, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 85.029, "latency": 35.102, "stderr": 0.858, "cost_per_test": 0.006419, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 84.335, "latency": 110.011, "stderr": 0.874, "cost_per_test": 0.019563, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 84.162, "latency": 111.943, "stderr": 0.878, "cost_per_test": 0.017413, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 83.931, "latency": 43.483, "stderr": 0.883, "cost_per_test": 0.012642, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 83.873, "latency": 119.005, "stderr": 0.884, "cost_per_test": 0.116094, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 83.584, "latency": 7.799, "stderr": 0.891, "cost_per_test": 0.006577, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 83.584, "latency": 131.728, "stderr": 0.891, "cost_per_test": 0.144122, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 83.468, "latency": 44.461, "stderr": 0.893, "cost_per_test": 0.026669, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 83.179, "latency": 75.307, "stderr": 0.899, "cost_per_test": 0.002764, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 83.064, "latency": 150.948, "stderr": 0.902, "cost_per_test": 0.02182, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 83.006, "latency": 18.817, "stderr": 0.903, "cost_per_test": 0.033133, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 82.948, "latency": 75.28, "stderr": 0.904, "cost_per_test": 0.126986, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 82.486, "latency": 12.448, "stderr": 0.914, "cost_per_test": 0.003187, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 81.908, "latency": 45.836, "stderr": 0.926, "cost_per_test": 0.002615, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 81.503, "latency": 72.114, "stderr": 0.934, "cost_per_test": 0.032532, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 81.34, "latency": 24.145, "stderr": 0.936, "cost_per_test": 0.007771, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 81.156, "latency": 68.354, "stderr": 0.94, "cost_per_test": 0.009431, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 81.098, "latency": 19.557, "stderr": 0.941, "cost_per_test": 0.021549, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 80.751, "latency": 27.866, "stderr": 0.948, "cost_per_test": 0.001858, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 80.416, "latency": 39.275, "stderr": 0.954, "cost_per_test": 0.015444, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 80.0, "latency": 42.47, "stderr": 0.962, "cost_per_test": 0.001396, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 79.665, "latency": 19.718, "stderr": 0.967, "cost_per_test": 0.010516, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 79.48, "latency": 25.643, "stderr": 0.971, "cost_per_test": 0.00187, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 79.306, "latency": 56.361, "stderr": 0.974, "cost_per_test": 0.058232, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 79.249, "latency": 50.813, "stderr": 0.975, "cost_per_test": 0.007061, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 78.914, "latency": 26.703, "stderr": 0.98, "cost_per_test": 0.007152, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 78.497, "latency": 123.645, "stderr": 0.988, "cost_per_test": 0.031466, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 77.514, "latency": 2.446, "stderr": 1.004, "cost_per_test": 0.168667, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 77.412, "latency": 26.415, "stderr": 1.005, "cost_per_test": 0.14586, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-0709": { "accuracy": 76.27, "latency": 102.831, "stderr": 1.022, "cost_per_test": 0.097579, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 75.434, "latency": 25.728, "stderr": 1.035, "cost_per_test": 0.000356, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 75.101, "latency": 69.581, "stderr": 1.039, "cost_per_test": 0.051028, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 74.928, "latency": 48.854, "stderr": 1.042, "cost_per_test": 0.050728, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 73.715, "latency": 46.277, "stderr": 1.058, "cost_per_test": 0.044475, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 73.584, "latency": 21.907, "stderr": 1.06, "cost_per_test": 0.000844, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 73.31, "latency": 14.023, "stderr": 1.063, "cost_per_test": 0.061801, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 72.775, "latency": 19.482, "stderr": 1.07, "cost_per_test": 0.00288, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 72.659, "latency": 96.354, "stderr": 1.072, "cost_per_test": 0.00056, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 72.543, "latency": 20.069, "stderr": 1.073, "cost_per_test": 0.000899, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 72.386, "latency": 10.823, "stderr": 1.075, "cost_per_test": 0.012845, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 72.386, "latency": 12.0, "stderr": 1.075, "cost_per_test": 0.007959, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-04-17-thinking": { "accuracy": 71.924, "latency": 471.866, "stderr": 1.08, "cost_per_test": 0.002641, "temperature": 0.1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 71.693, "latency": 7.908, "stderr": 1.083, "cost_per_test": 0.001078, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 71.519, "latency": 8.323, "stderr": 1.085, "cost_per_test": 0.012358, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 70.942, "latency": 30.776, "stderr": 1.091, "cost_per_test": 0.002619, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 70.537, "latency": 8.225, "stderr": 1.096, "cost_per_test": 0.001788, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 69.786, "latency": 45.203, "stderr": 1.104, "cost_per_test": 0.000365, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 68.804, "latency": 46.011, "stderr": 1.114, "cost_per_test": 0.010237, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 67.995, "latency": 37.103, "stderr": 1.121, "cost_per_test": 0.02536, "temperature": 0.1, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 66.185, "latency": 13.55, "stderr": 1.137, "cost_per_test": 0.001477, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 65.511, "latency": 43.736, "stderr": 1.142, "cost_per_test": 0.001846, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 65.202, "latency": 19.095, "stderr": 1.145, "cost_per_test": 0.004413, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 64.566, "latency": 42.144, "stderr": 1.15, "cost_per_test": 0.021457, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 64.009, "latency": 47.52, "stderr": 1.154, "cost_per_test": 0.006251, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 63.699, "latency": 51.426, "stderr": 1.156, "cost_per_test": 0.000628, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 63.41, "latency": 7.209, "stderr": 1.158, "cost_per_test": 0.001555, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 62.969, "latency": 12.56, "stderr": 1.161, "cost_per_test": 0.001567, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 62.161, "latency": 45.873, "stderr": 1.166, "cost_per_test": 0.006822, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 61.792, "latency": 714.648, "stderr": 1.168, "cost_per_test": 0.030117, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 60.081, "latency": 13.401, "stderr": 1.177, "cost_per_test": 0.000265, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 58.752, "latency": 10.506, "stderr": 1.183, "cost_per_test": 0.000835, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-2-vision-1212": { "accuracy": 57.25, "latency": 18.355, "stderr": 1.189, "cost_per_test": 0.005402, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 57.192, "latency": 41.314, "stderr": 1.189, "cost_per_test": 0.000114, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 56.557, "latency": 45.016, "stderr": 1.191, "cost_per_test": 0.00037, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 55.055, "latency": 7.251, "stderr": 1.196, "cost_per_test": 0.000566, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo": { "accuracy": 48.065, "latency": 56.329, "stderr": 1.201, "cost_per_test": 0.008287, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 46.069, "latency": 40.481, "stderr": 1.198, "cost_per_test": 0.027476, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo": { "accuracy": 38.821, "latency": 38.07, "stderr": 1.171, "cost_per_test": 0.000171, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 22.775, "latency": 135.183, "stderr": 1.008, "cost_per_test": 0.01513, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null } } } }