{ "metadata": { "benchmark": "MMMU Pro", "slug": "mmmu", "description": "Multimodal Multi-task Benchmark", "benchmark_id": "mmmu", "family": "mmmu", "version": "1", "updated": "2026-09-01", "dataset_type": "public", "industry": "academic", "tasks": { "overall": "Overall" }, "models": [ "alibaba/qwen3.5-flash", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-plus", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "anthropic/claude-3-5-sonnet-20241022", "anthropic/claude-3-7-sonnet-20250219", "anthropic/claude-3-7-sonnet-20250219-thinking", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-1-20250805-thinking", "anthropic/claude-opus-4-20250514", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "fireworks/llama4-maverick-instruct-basic", "google/gemini-1.5-flash-002", "google/gemini-1.5-pro-002", "google/gemini-2.0-flash-001", "google/gemini-2.5-flash-lite-preview-09-2025", "google/gemini-2.5-flash-lite-preview-09-2025-thinking", "google/gemini-2.5-flash-preview-04-17", "google/gemini-2.5-flash-preview-04-17-thinking", "google/gemini-2.5-flash-preview-09-2025", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-pro-exp-03-25", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "grok/grok-2-vision-1212", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse_spark", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "minimax/MiniMax-M3", "mistralai/magistral-medium-2509", "mistralai/magistral-small-2509", "mistralai/mistral-large-2512", "mistralai/mistral-medium-2505", "mistralai/mistral-small-2503", "openai/gpt-4.1-2025-04-14", "openai/gpt-4.1-mini-2025-04-14", "openai/gpt-4.1-nano-2025-04-14", "openai/gpt-4o-2024-08-06", "openai/gpt-4o-2024-11-20", "openai/gpt-4o-mini-2024-07-18", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-nano-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/o1-2024-12-17", "openai/o3-2025-04-16", "openai/o4-mini-2025-04-16", "thinkingmachines/inkling", "together/meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo", "together/meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo", "together/meta-llama/Llama-4-Scout-17B-16E-Instruct", "xiaomi/mimo-v2.5", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "custom", "mode": "one-shot", "archived": false, "partner": false, "total_models": 93 }, "tasks": { "overall": { "anthropic/claude-fable-5-1": { "accuracy": 90.636, "latency": 37.47, "stderr": 0.7, "cost_per_test": 0.166285, "avg_input_tokens": 1964.9, "avg_output_tokens": 2834.7, "avg_cache_write_tokens": 1960.9, "turn_metrics": { "tasks_measured": 1726, "avg_turns": 1.0, "avg_query_seconds": 37.5, "median_query_seconds": 11.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 89.884, "latency": 30.028, "stderr": 0.725, "cost_per_test": 0.069014, "avg_input_tokens": 1962.8, "avg_output_tokens": 2269.9, "avg_reasoning_tokens": 1861.7, "avg_cache_write_tokens": 1960.8, "turn_metrics": { "tasks_measured": 1726, "avg_turns": 1.0, "avg_query_seconds": 30.0, "median_query_seconds": 9.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 89.306, "latency": 61.44, "stderr": 0.743, "cost_per_test": 0.16064, "avg_input_tokens": 1958.0, "avg_output_tokens": 2799.0, "avg_cache_write_tokens": 667.6, "turn_metrics": { "tasks_measured": 1724, "avg_turns": 1.0, "avg_query_seconds": 61.4, "median_query_seconds": 24.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 89.075, "latency": 12.028, "stderr": 0.75, "cost_per_test": 0.004472, "avg_input_tokens": 1399.5, "avg_output_tokens": 3834.7, "avg_reasoning_tokens": 3518.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 12.0, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 88.96, "latency": 6.283, "stderr": 0.753, "cost_per_test": 0.014052, "avg_input_tokens": 1400.2, "avg_output_tokens": 1593.5, "avg_reasoning_tokens": 1312.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 6.3, "median_query_seconds": 4.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 88.844, "latency": 26.198, "stderr": 0.757, "cost_per_test": 0.055795, "avg_input_tokens": 2042.9, "avg_output_tokens": 1520.9, "avg_reasoning_tokens": 1441.4, "avg_cache_read_tokens": 10.1, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 26.2, "median_query_seconds": 6.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 88.382, "latency": 11.313, "stderr": 0.77, "cost_per_test": 0.0186, "avg_input_tokens": 1400.2, "avg_output_tokens": 2200.0, "avg_reasoning_tokens": 1872.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 11.3, "median_query_seconds": 7.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 88.266, "latency": 12.226, "stderr": 0.774, "cost_per_test": 0.025155, "avg_input_tokens": 1400.2, "avg_output_tokens": 2563.4, "avg_reasoning_tokens": 2187.3, "avg_cache_read_tokens": 11.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 12.2, "median_query_seconds": 7.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 88.266, "latency": 54.147, "stderr": 0.774, "cost_per_test": 0.043319, "avg_input_tokens": 2005.7, "avg_output_tokens": 2073.1, "avg_reasoning_tokens": 1959.3, "avg_cache_read_tokens": 5.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 54.1, "median_query_seconds": 16.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 88.208, "latency": 76.99, "stderr": 0.775, "cost_per_test": 0.064084, "avg_input_tokens": 1401.0, "avg_output_tokens": 5106.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 77.0, "median_query_seconds": 34.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 88.15, "latency": 106.516, "stderr": 0.777, "cost_per_test": 0.059115, "avg_input_tokens": 1995.5, "avg_output_tokens": 3865.2, "avg_reasoning_tokens": 3728.3, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 106.5, "median_query_seconds": 40.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 88.035, "latency": 43.853, "stderr": 0.78, "cost_per_test": 0.015881, "avg_input_tokens": 1491.7, "avg_output_tokens": 2164.5, "avg_reasoning_tokens": 1889.5, "avg_cache_read_tokens": 51.1, "turn_metrics": { "tasks_measured": 1722, "avg_turns": 1.0, "avg_query_seconds": 43.9, "median_query_seconds": 22.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 87.63, "latency": 27.856, "stderr": 0.792, "cost_per_test": 0.014422, "avg_input_tokens": 1400.2, "avg_output_tokens": 4574.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 27.9, "median_query_seconds": 11.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 87.514, "latency": 48.102, "stderr": 0.795, "cost_per_test": 0.010159, "avg_input_tokens": 1400.4, "avg_output_tokens": 613.2, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 48.1, "median_query_seconds": 30.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 87.514, "latency": 65.565, "stderr": 0.795, "cost_per_test": 0.047083, "avg_input_tokens": 1845.3, "avg_output_tokens": 2836.3, "avg_reasoning_tokens": 2688.1, "avg_cache_read_tokens": 33.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 65.6, "median_query_seconds": 20.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 87.399, "latency": 48.272, "stderr": 0.798, "cost_per_test": 0.001746, "avg_input_tokens": 1961.1, "avg_output_tokens": 2420.3, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 48.3, "median_query_seconds": 23.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 86.667, "latency": 96.324, "stderr": 0.822, "cost_per_test": 0.061121, "avg_input_tokens": 1869.8, "avg_output_tokens": 4134.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1710, "avg_turns": 1.0, "avg_query_seconds": 96.3, "median_query_seconds": 25.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 86.59, "latency": 31.715, "stderr": 0.819, "cost_per_test": 0.01361, "avg_input_tokens": 2061.4, "avg_output_tokens": 2657.2, "avg_reasoning_tokens": 2466.6, "avg_cache_read_tokens": 149.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 31.7, "median_query_seconds": 18.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 86.59, "latency": 49.044, "stderr": 0.819, "cost_per_test": 0.120663, "avg_input_tokens": 1961.5, "avg_output_tokens": 4430.3, "avg_cache_write_tokens": 413.6, "turn_metrics": { "tasks_measured": 1724, "avg_turns": 1.0, "avg_query_seconds": 49.0, "median_query_seconds": 12.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 86.474, "latency": 13.033, "stderr": 0.822, "cost_per_test": 0.016375, "avg_input_tokens": 1845.3, "avg_output_tokens": 1058.9, "avg_reasoning_tokens": 971.9, "avg_cache_read_tokens": 12.3, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 13.0, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 86.301, "latency": 241.908, "stderr": 0.827, "cost_per_test": 0.034867, "avg_input_tokens": 1911.3, "avg_output_tokens": 8296.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 94.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1728, "avg_turns": 1.0, "avg_query_seconds": 241.9, "median_query_seconds": 111.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 86.127, "latency": 73.839, "stderr": 0.831, "cost_per_test": 0.018022, "avg_input_tokens": 1908.1, "avg_output_tokens": 3680.6, "avg_reasoning_tokens": 3447.0, "avg_cache_read_tokens": 5.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1729, "avg_turns": 1.0, "avg_query_seconds": 73.8, "median_query_seconds": 31.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 86.012, "latency": 28.218, "stderr": 0.834, "cost_per_test": 0.000962, "avg_input_tokens": 1914.1, "avg_output_tokens": 3275.7, "avg_reasoning_tokens": 2867.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1724, "avg_turns": 1.0, "avg_query_seconds": 28.2, "median_query_seconds": 14.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 85.549, "latency": 70.801, "stderr": 0.845, "cost_per_test": 0.170461, "avg_input_tokens": 1961.8, "avg_output_tokens": 6405.7, "avg_cache_write_tokens": 406.7, "turn_metrics": { "tasks_measured": 1691, "avg_turns": 1.0, "avg_query_seconds": 70.8, "median_query_seconds": 13.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 85.029, "latency": 35.102, "stderr": 0.858, "cost_per_test": 0.006419, "avg_input_tokens": 1845.5, "avg_output_tokens": 5044.4, "avg_reasoning_tokens": 4947.6, "avg_cache_read_tokens": 0.8, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 35.1, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 81.156, "latency": 68.354, "stderr": 0.94, "cost_per_test": 0.009431, "avg_input_tokens": 2075.5, "avg_output_tokens": 3602.7, "avg_cache_read_tokens": 213.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 70.3, "median_query_seconds": 26.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 84.335, "latency": 110.011, "stderr": 0.874, "cost_per_test": 0.019563, "avg_input_tokens": 1910.8, "avg_output_tokens": 6145.7, "turn_metrics": { "tasks_measured": 1728, "avg_turns": 1.0, "avg_query_seconds": 110.0, "median_query_seconds": 68.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 84.162, "latency": 111.943, "stderr": 0.878, "cost_per_test": 0.017413, "avg_input_tokens": 1489.5, "avg_output_tokens": 5556.2, "avg_reasoning_tokens": 5026.9, "turn_metrics": { "tasks_measured": 1695, "avg_turns": 1.0, "avg_query_seconds": 111.9, "median_query_seconds": 61.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 83.931, "latency": 43.483, "stderr": 0.883, "cost_per_test": 0.012642, "avg_input_tokens": 1532.3, "avg_output_tokens": 3958.7, "avg_reasoning_tokens": 3728.5, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 43.5, "median_query_seconds": 18.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 83.873, "latency": 119.005, "stderr": 0.884, "cost_per_test": 0.116094, "avg_input_tokens": 1938.0, "avg_output_tokens": 7339.4, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 119.0, "median_query_seconds": 19.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 83.584, "latency": 7.799, "stderr": 0.891, "cost_per_test": 0.006577, "avg_input_tokens": 1400.2, "avg_output_tokens": 2462.6, "avg_reasoning_tokens": 2166.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 7.8, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 83.584, "latency": 131.728, "stderr": 0.891, "cost_per_test": 0.144122, "avg_input_tokens": 1938.0, "avg_output_tokens": 9200.7, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 131.7, "median_query_seconds": 27.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 83.468, "latency": 44.461, "stderr": 0.893, "cost_per_test": 0.026669, "avg_input_tokens": 1677.3, "avg_output_tokens": 3906.0, "avg_reasoning_tokens": 3629.4, "avg_cache_read_tokens": 67.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 44.5, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 83.179, "latency": 75.307, "stderr": 0.899, "cost_per_test": 0.002764, "avg_input_tokens": 858.5, "avg_output_tokens": 169.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 75.3, "median_query_seconds": 31.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 83.064, "latency": 150.948, "stderr": 0.902, "cost_per_test": 0.02182, "avg_input_tokens": 1566.2, "avg_output_tokens": 7946.2, "avg_reasoning_tokens": 7681.8, "avg_cache_read_tokens": 3.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 150.9, "median_query_seconds": 105.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 83.006, "latency": 18.817, "stderr": 0.903, "cost_per_test": 0.033133, "avg_input_tokens": 1961.3, "avg_output_tokens": 1718.6, "avg_cache_write_tokens": 1959.3, "turn_metrics": { "tasks_measured": 1725, "avg_turns": 1.0, "avg_query_seconds": 18.8, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 82.948, "latency": 75.28, "stderr": 0.904, "cost_per_test": 0.126986, "avg_input_tokens": 1698.9, "avg_output_tokens": 4739.6, "turn_metrics": { "tasks_measured": 1729, "avg_turns": 1.0, "avg_query_seconds": 75.3, "median_query_seconds": 25.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 82.486, "latency": 12.448, "stderr": 0.914, "cost_per_test": 0.003187, "avg_input_tokens": 1411.8, "avg_output_tokens": 3916.8, "avg_reasoning_tokens": 3502.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 12.4, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 81.908, "latency": 45.836, "stderr": 0.926, "cost_per_test": 0.002615, "avg_input_tokens": 1493.3, "avg_output_tokens": 6164.6, "avg_reasoning_tokens": 5649.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1712, "avg_turns": 1.0, "avg_query_seconds": 45.8, "median_query_seconds": 25.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 81.503, "latency": 72.114, "stderr": 0.934, "cost_per_test": 0.032532, "avg_input_tokens": 790.6, "avg_output_tokens": 3167.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1725, "avg_turns": 1.0, "avg_query_seconds": 72.3, "median_query_seconds": 35.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 81.34, "latency": 24.145, "stderr": 0.936, "cost_per_test": 0.007771, "avg_input_tokens": 432.6, "avg_output_tokens": 723.1, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 24.1, "median_query_seconds": 13.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 81.098, "latency": 19.557, "stderr": 0.941, "cost_per_test": 0.021549, "avg_input_tokens": 1670.8, "avg_output_tokens": 527.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 19.6, "median_query_seconds": 15.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 80.751, "latency": 27.866, "stderr": 0.948, "cost_per_test": 0.001858, "avg_input_tokens": 1381.4, "avg_output_tokens": 577.6, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 27.9, "median_query_seconds": 24.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 80.416, "latency": 39.275, "stderr": 0.954, "cost_per_test": 0.015444, "avg_input_tokens": 934.9, "avg_output_tokens": 1696.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 39.3, "median_query_seconds": 16.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 80.0, "latency": 42.47, "stderr": 0.962, "cost_per_test": 0.001396, "avg_input_tokens": 1727.5, "avg_output_tokens": 4236.4, "avg_reasoning_tokens": 3510.6, "avg_cache_read_tokens": 235.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 42.5, "median_query_seconds": 17.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 79.665, "latency": 19.718, "stderr": 0.967, "cost_per_test": 0.010516, "avg_input_tokens": 2586.6, "avg_output_tokens": 1743.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 19.7, "median_query_seconds": 10.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 79.48, "latency": 25.643, "stderr": 0.971, "cost_per_test": 0.00187, "avg_input_tokens": 1381.4, "avg_output_tokens": 582.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 25.6, "median_query_seconds": 22.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 79.306, "latency": 56.361, "stderr": 0.974, "cost_per_test": 0.058232, "avg_input_tokens": 1949.0, "avg_output_tokens": 3492.3, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 56.4, "median_query_seconds": 22.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 79.249, "latency": 50.813, "stderr": 0.975, "cost_per_test": 0.007061, "avg_input_tokens": 1845.3, "avg_output_tokens": 3302.8, "avg_reasoning_tokens": 3193.5, "avg_cache_read_tokens": 28.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 50.8, "median_query_seconds": 15.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 78.914, "latency": 26.703, "stderr": 0.98, "cost_per_test": 0.007152, "avg_input_tokens": 565.8, "avg_output_tokens": 3505.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 26.7, "median_query_seconds": 11.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 78.497, "latency": 123.645, "stderr": 0.988, "cost_per_test": 0.031466, "avg_input_tokens": 1059.9, "avg_output_tokens": 7527.0, "avg_cache_read_tokens": 78.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 123.6, "median_query_seconds": 30.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 77.514, "latency": 2.446, "stderr": 1.004, "cost_per_test": 0.168667, "avg_input_tokens": 1949.0, "avg_output_tokens": 1859.1, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 2.4, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 77.412, "latency": 26.415, "stderr": 1.005, "cost_per_test": 0.14586, "avg_input_tokens": 1023.3, "avg_output_tokens": 2175.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 26.4, "median_query_seconds": 15.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-0709": { "accuracy": 76.27, "latency": 102.831, "stderr": 1.022, "cost_per_test": 0.097579, "avg_input_tokens": 1690.8, "avg_output_tokens": 6167.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 102.8, "median_query_seconds": 51.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 75.434, "latency": 25.728, "stderr": 1.035, "cost_per_test": 0.000356, "avg_input_tokens": 1381.4, "avg_output_tokens": 544.9, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 25.7, "median_query_seconds": 22.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 75.101, "latency": 69.581, "stderr": 1.039, "cost_per_test": 0.051028, "avg_input_tokens": 1949.8, "avg_output_tokens": 3011.9, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 69.6, "median_query_seconds": 19.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 74.928, "latency": 48.854, "stderr": 1.042, "cost_per_test": 0.050728, "avg_input_tokens": 1948.8, "avg_output_tokens": 2992.1, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 48.9, "median_query_seconds": 23.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 73.715, "latency": 46.277, "stderr": 1.058, "cost_per_test": 0.044475, "avg_input_tokens": 683.4, "avg_output_tokens": 456.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 46.3, "median_query_seconds": 16.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 73.584, "latency": 21.907, "stderr": 1.06, "cost_per_test": 0.000844, "avg_input_tokens": 1845.3, "avg_output_tokens": 1884.7, "avg_reasoning_tokens": 1722.6, "avg_cache_read_tokens": 40.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 21.9, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 73.31, "latency": 14.023, "stderr": 1.063, "cost_per_test": 0.061801, "avg_input_tokens": 1920.8, "avg_output_tokens": 439.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 14.0, "median_query_seconds": 12.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 72.775, "latency": 19.482, "stderr": 1.07, "cost_per_test": 0.00288, "avg_input_tokens": 1732.7, "avg_output_tokens": 5066.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 19.5, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 72.659, "latency": 96.354, "stderr": 1.072, "cost_per_test": 0.00056, "avg_input_tokens": 1585.9, "avg_output_tokens": 485.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 96.4, "median_query_seconds": 33.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 72.543, "latency": 20.069, "stderr": 1.073, "cost_per_test": 0.000899, "avg_input_tokens": 1381.4, "avg_output_tokens": 1901.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 20.1, "median_query_seconds": 19.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 72.386, "latency": 10.823, "stderr": 1.075, "cost_per_test": 0.012845, "avg_input_tokens": 1920.8, "avg_output_tokens": 472.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 10.8, "median_query_seconds": 9.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 72.386, "latency": 12.0, "stderr": 1.075, "cost_per_test": 0.007959, "avg_input_tokens": 1024.3, "avg_output_tokens": 738.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 12.0, "median_query_seconds": 6.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-04-17-thinking": { "accuracy": 71.924, "latency": 471.866, "stderr": 1.08, "cost_per_test": 0.002641, "avg_input_tokens": 432.7, "avg_output_tokens": 1004.3, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 471.9, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.1, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 71.693, "latency": 7.908, "stderr": 1.083, "cost_per_test": 0.001078, "avg_input_tokens": 2607.2, "avg_output_tokens": 572.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 7.9, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 71.519, "latency": 8.323, "stderr": 1.085, "cost_per_test": 0.012358, "avg_input_tokens": 1920.8, "avg_output_tokens": 439.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 8.3, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 70.942, "latency": 30.776, "stderr": 1.091, "cost_per_test": 0.002619, "avg_input_tokens": 565.8, "avg_output_tokens": 6477.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 30.8, "median_query_seconds": 20.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 70.537, "latency": 8.225, "stderr": 1.096, "cost_per_test": 0.001788, "avg_input_tokens": 2445.6, "avg_output_tokens": 506.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 8.2, "median_query_seconds": 5.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 69.786, "latency": 45.203, "stderr": 1.104, "cost_per_test": 0.000365, "avg_input_tokens": 1411.9, "avg_output_tokens": 560.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 45.2, "median_query_seconds": 19.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 68.804, "latency": 46.011, "stderr": 1.114, "cost_per_test": 0.010237, "avg_input_tokens": 1920.8, "avg_output_tokens": 298.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 46.0, "median_query_seconds": 19.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 67.995, "latency": 37.103, "stderr": 1.121, "cost_per_test": 0.02536, "avg_input_tokens": 432.7, "avg_output_tokens": 10092.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 37.1, "median_query_seconds": 4.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.1, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 66.185, "latency": 13.55, "stderr": 1.137, "cost_per_test": 0.001477, "avg_input_tokens": 1953.0, "avg_output_tokens": 333.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 13.6, "median_query_seconds": 11.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 65.511, "latency": 43.736, "stderr": 1.142, "cost_per_test": 0.001846, "avg_input_tokens": 431.5, "avg_output_tokens": 261.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 43.7, "median_query_seconds": 18.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 65.202, "latency": 19.095, "stderr": 1.145, "cost_per_test": 0.004413, "avg_input_tokens": 1953.1, "avg_output_tokens": 2292.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 19.1, "median_query_seconds": 9.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 64.566, "latency": 42.144, "stderr": 1.15, "cost_per_test": 0.021457, "avg_input_tokens": 1953.9, "avg_output_tokens": 3524.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 42.1, "median_query_seconds": 16.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 64.009, "latency": 47.52, "stderr": 1.154, "cost_per_test": 0.006251, "avg_input_tokens": 1024.3, "avg_output_tokens": 369.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 47.5, "median_query_seconds": 19.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 63.699, "latency": 51.426, "stderr": 1.156, "cost_per_test": 0.000628, "avg_input_tokens": 1569.6, "avg_output_tokens": 628.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 51.4, "median_query_seconds": 17.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 63.41, "latency": 7.209, "stderr": 1.158, "cost_per_test": 0.001555, "avg_input_tokens": 1742.7, "avg_output_tokens": 2413.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 7.2, "median_query_seconds": 3.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 62.969, "latency": 12.56, "stderr": 1.161, "cost_per_test": 0.001567, "avg_input_tokens": 1952.0, "avg_output_tokens": 393.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 12.6, "median_query_seconds": 7.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 62.161, "latency": 45.873, "stderr": 1.166, "cost_per_test": 0.006822, "avg_input_tokens": 1023.8, "avg_output_tokens": 426.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 45.9, "median_query_seconds": 19.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 61.792, "latency": 714.648, "stderr": 1.168, "cost_per_test": 0.030117, "avg_input_tokens": 1554.6, "avg_output_tokens": 4504.8, "avg_reasoning_tokens": 4257.5, "avg_cache_read_tokens": 13.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 714.6, "median_query_seconds": 695.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 60.081, "latency": 13.401, "stderr": 1.177, "cost_per_test": 0.000265, "avg_input_tokens": 1952.0, "avg_output_tokens": 393.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 13.4, "median_query_seconds": 8.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 58.752, "latency": 10.506, "stderr": 1.183, "cost_per_test": 0.000835, "avg_input_tokens": 2605.7, "avg_output_tokens": 620.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 10.5, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-2-vision-1212": { "accuracy": 57.25, "latency": 18.355, "stderr": 1.189, "cost_per_test": 0.005402, "avg_input_tokens": 435.2, "avg_output_tokens": 453.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 18.4, "median_query_seconds": 13.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 57.192, "latency": 41.314, "stderr": 1.189, "cost_per_test": 0.000114, "avg_input_tokens": 431.5, "avg_output_tokens": 271.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 41.3, "median_query_seconds": 18.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 56.557, "latency": 45.016, "stderr": 1.191, "cost_per_test": 0.00037, "avg_input_tokens": 1023.4, "avg_output_tokens": 361.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 45.0, "median_query_seconds": 19.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 55.055, "latency": 7.251, "stderr": 1.196, "cost_per_test": 0.000566, "avg_input_tokens": 3643.4, "avg_output_tokens": 503.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 7.3, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo": { "accuracy": 48.065, "latency": 56.329, "stderr": 1.201, "cost_per_test": 0.008287, "avg_input_tokens": 6545.2, "avg_output_tokens": 360.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 56.3, "median_query_seconds": 25.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 46.069, "latency": 40.481, "stderr": 1.198, "cost_per_test": 0.027476, "avg_input_tokens": 1951.2, "avg_output_tokens": 5156.3, "turn_metrics": { "tasks_measured": 1714, "avg_turns": 1.0, "avg_query_seconds": 40.9, "median_query_seconds": 15.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo": { "accuracy": 38.821, "latency": 38.07, "stderr": 1.171, "cost_per_test": 0.000171, "avg_input_tokens": 141.4, "avg_output_tokens": 811.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1730, "avg_turns": 1.0, "avg_query_seconds": 38.1, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 22.775, "latency": 135.183, "stderr": 1.008, "cost_per_test": 0.01513, "avg_input_tokens": 1492.7, "avg_output_tokens": 6055.2, "turn_metrics": { "tasks_measured": 1720, "avg_turns": 1.0, "avg_query_seconds": 135.2, "median_query_seconds": 89.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null } } } }