{ "metadata": { "benchmark": "AIME", "slug": "aime", "description": "Challenging national math exam given to top high-school students", "benchmark_id": "aime", "family": "aime", "version": "1", "updated": "2026-04-16", "dataset_type": "public", "industry": "math", "tasks": { "overall": "Overall", "aime_2024": "AIME 2024", "aime_2025": "AIME 2025" }, "models": [ "ai21labs/jamba-large-1.6", "ai21labs/jamba-mini-1.6", "alibaba/qwen3-max", "alibaba/qwen3-max-preview", "alibaba/qwen3.5-flash", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-plus", "anthropic/claude-3-5-haiku-20241022", "anthropic/claude-3-5-sonnet-20241022", "anthropic/claude-3-7-sonnet-20250219", "anthropic/claude-3-7-sonnet-20250219-thinking", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-1-20250805-thinking", "anthropic/claude-opus-4-20250514", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "cohere/command-a-03-2025", "fireworks/deepseek-r1", "fireworks/deepseek-v3", "fireworks/deepseek-v3-0324", "fireworks/deepseek-v3p2", "fireworks/deepseek-v3p2-thinking", "fireworks/gpt-oss-120b", "fireworks/gpt-oss-20b", "fireworks/llama4-maverick-instruct-basic", "fireworks/qwen3-235b-a22b", "google/gemini-1.5-flash-002", "google/gemini-1.5-pro-002", "google/gemini-2.0-flash-001", "google/gemini-2.5-flash-lite-preview-09-2025", "google/gemini-2.5-flash-lite-preview-09-2025-thinking", "google/gemini-2.5-flash-preview-09-2025", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-pro-exp-03-25", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "grok/grok-2-1212", "grok/grok-3", "grok/grok-3-mini-fast-high-reasoning", "grok/grok-3-mini-fast-low-reasoning", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "kimi/kimi-k2-thinking", "kimi/kimi-k2.5-thinking", "meta/muse_spark", "minimax/MiniMax-M2.1", "minimax/MiniMax-M2.5", "minimax/MiniMax-M2.7", "mistralai/magistral-medium-2509", "mistralai/magistral-small-2509", "mistralai/mistral-large-2411", "mistralai/mistral-large-2512", "mistralai/mistral-medium-2505", "mistralai/mistral-small-2402", "mistralai/mistral-small-2503", "openai/gpt-4.1-2025-04-14", "openai/gpt-4.1-mini-2025-04-14", "openai/gpt-4.1-nano-2025-04-14", "openai/gpt-4o-2024-08-06", "openai/gpt-4o-2024-11-20", "openai/gpt-4o-mini-2024-07-18", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-nano-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/o1-2024-12-17", "openai/o3-2025-04-16", "openai/o3-mini-2025-01-31", "openai/o4-mini-2025-04-16", "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561", "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking", "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "together/meta-llama/Llama-4-Scout-17B-16E-Instruct", "together/moonshotai/Kimi-K2-Instruct", "zai/glm-4.5", "zai/glm-4.6", "zai/glm-4.7", "zai/glm-5-thinking", "zai/glm-5.1" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "platform", "mode": "one-shot", "archived": true, "partner": false, "total_models": 96 }, "tasks": { "overall": { "google/gemini-3.1-pro-preview": { "accuracy": 98.125, "latency": 104.525, "stderr": 0.343, "cost_per_test": 0.126991, "avg_input_tokens": 128.0, "avg_output_tokens": 10561.3, "avg_reasoning_tokens": 9606.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 104.5, "median_query_seconds": 76.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 96.875, "latency": 120.31, "stderr": 0.208, "cost_per_test": 0.135323, "avg_input_tokens": 359.8, "avg_output_tokens": 11498.0, "avg_reasoning_tokens": 9115.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 120.3, "median_query_seconds": 56.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 96.875, "latency": 156.746, "stderr": 0.378, "cost_per_test": 0.004561, "avg_input_tokens": 180.9, "avg_output_tokens": 7556.6, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 156.8, "median_query_seconds": 112.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 96.68, "latency": 109.264, "stderr": 0.384, "cost_per_test": 0.169306, "avg_input_tokens": 140.6, "avg_output_tokens": 14085.4, "avg_reasoning_tokens": 13133.4, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 109.3, "median_query_seconds": 103.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 96.667, "latency": 121.466, "stderr": 0.534, "cost_per_test": 0.138186, "avg_input_tokens": 130.4, "avg_output_tokens": 9190.6, "avg_reasoning_tokens": 8536.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 121.5, "median_query_seconds": 66.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 96.458, "latency": 106.799, "stderr": 0.516, "cost_per_test": 0.059397, "avg_input_tokens": 244.0, "avg_output_tokens": 9875.2, "avg_reasoning_tokens": 9038.6, "avg_cache_read_tokens": 190.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 106.8, "median_query_seconds": 93.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 96.25, "latency": 182.065, "stderr": 0.522, "cost_per_test": 0.450513, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 95.625, "latency": 84.007, "stderr": 0.516, "cost_per_test": 0.046328, "avg_input_tokens": 128.0, "avg_output_tokens": 15421.2, "avg_reasoning_tokens": 14595.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 84.0, "median_query_seconds": 87.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 95.625, "latency": 295.678, "stderr": 0.644, "cost_per_test": 0.343821, "avg_input_tokens": 162.7, "avg_output_tokens": 22888.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 295.7, "median_query_seconds": 147.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 95.625, "latency": 300.051, "stderr": 0.409, "cost_per_test": 0.128636, "avg_input_tokens": 152.1, "avg_output_tokens": 28560.5, "avg_reasoning_tokens": 28089.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 300.0, "median_query_seconds": 254.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 95.625, "latency": 322.611, "stderr": 0.343, "cost_per_test": 0.064142, "avg_input_tokens": 128.6, "avg_output_tokens": 21373.8, "avg_cache_read_tokens": 111.6, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 322.6, "median_query_seconds": 270.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 95.417, "latency": 212.201, "stderr": 0.369, "cost_per_test": 1.320246, "avg_input_tokens": 174.7, "avg_output_tokens": 17568.3, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 212.2, "median_query_seconds": 200.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 94.583, "latency": 516.49, "stderr": 0.579, "cost_per_test": 0.09352, "avg_input_tokens": 138.4, "avg_output_tokens": 31150.0, "avg_reasoning_tokens": 30306.6, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 516.5, "median_query_seconds": 455.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 93.374, "latency": 292.202, "stderr": 1.305, "cost_per_test": 0.092693, "avg_input_tokens": 895.3, "avg_output_tokens": 9322.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 292.2, "median_query_seconds": 111.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 93.333, "latency": 163.891, "stderr": 0.62, "cost_per_test": 0.150354, "avg_input_tokens": 130.4, "avg_output_tokens": 15019.1, "avg_reasoning_tokens": 13468.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 163.9, "median_query_seconds": 98.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.7": { "accuracy": 93.333, "latency": 529.192, "stderr": 0.445, "cost_per_test": 0.043308, "avg_input_tokens": 129.7, "avg_output_tokens": 19690.9, "avg_cache_read_tokens": 22.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 529.2, "median_query_seconds": 368.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-4.6": { "accuracy": 92.708, "latency": 699.868, "stderr": 0.605, "cost_per_test": 0.116286, "avg_input_tokens": 131.4, "avg_output_tokens": 16617.3, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 699.8, "median_query_seconds": 208.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 92.598, "latency": 156.629, "stderr": 1.225, "cost_per_test": 0.010472, "avg_input_tokens": 198.4, "avg_output_tokens": 17554.2, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 156.6, "median_query_seconds": 111.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 92.5, "latency": 239.603, "stderr": 0.891, "cost_per_test": 0.011117, "avg_input_tokens": 138.4, "avg_output_tokens": 27758.5, "avg_reasoning_tokens": 26437.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 239.6, "median_query_seconds": 181.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 92.292, "latency": 328.034, "stderr": 0.438, "cost_per_test": 0.436012, "avg_input_tokens": 162.7, "avg_output_tokens": 29034.9, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 328.0, "median_query_seconds": 211.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 91.875, "latency": 91.997, "stderr": 0.584, "cost_per_test": 0.004094, "avg_input_tokens": 274.9, "avg_output_tokens": 8150.6, "avg_reasoning_tokens": 7596.6, "avg_cache_read_tokens": 240.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 92.0, "median_query_seconds": 56.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.1": { "accuracy": 91.875, "latency": 375.201, "stderr": 0.438, "cost_per_test": 0.070948, "avg_input_tokens": 129.6, "avg_output_tokens": 22140.3, "avg_reasoning_tokens": 21644.8, "avg_cache_read_tokens": 38.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 375.2, "median_query_seconds": 307.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 91.667, "latency": 458.694, "stderr": 0.772, "cost_per_test": 0.069817, "avg_input_tokens": 129.6, "avg_output_tokens": 21852.6, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 458.7, "median_query_seconds": 351.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 91.458, "latency": 105.53, "stderr": 0.836, "cost_per_test": 0.02276, "avg_input_tokens": 130.4, "avg_output_tokens": 11363.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 105.6, "median_query_seconds": 41.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 91.25, "latency": 34.559, "stderr": 1.033, "cost_per_test": 0.004317, "avg_input_tokens": 235.9, "avg_output_tokens": 8540.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 34.5, "median_query_seconds": 18.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 91.042, "latency": 236.637, "stderr": 0.734, "cost_per_test": 0.022959, "avg_input_tokens": 140.0, "avg_output_tokens": 19121.8, "avg_cache_read_tokens": 115.4, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 236.6, "median_query_seconds": 147.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4-0709": { "accuracy": 90.556, "latency": 133.228, "stderr": 1.757, "cost_per_test": 0.174399, "avg_input_tokens": 125.9, "avg_output_tokens": 11601.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 133.2, "median_query_seconds": 79.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 88.75, "latency": 65.876, "stderr": 0.99, "cost_per_test": 0.003432, "avg_input_tokens": 279.0, "avg_output_tokens": 8544.1, "avg_reasoning_tokens": 7693.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 65.9, "median_query_seconds": 27.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 88.75, "latency": 373.487, "stderr": 0.877, "cost_per_test": 0.041379, "avg_input_tokens": 160.0, "avg_output_tokens": 17226.0, "avg_cache_read_tokens": 67.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 373.5, "median_query_seconds": 227.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 88.19, "latency": 217.504, "stderr": 0.686, "cost_per_test": 0.266927, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.5": { "accuracy": 86.667, "latency": 333.117, "stderr": 1.332, "cost_per_test": 0.04689, "avg_input_tokens": 131.6, "avg_output_tokens": 21278.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 333.2, "median_query_seconds": 276.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 86.458, "latency": 154.65, "stderr": 1.054, "cost_per_test": 0.050817, "avg_input_tokens": 130.4, "avg_output_tokens": 11516.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 154.6, "median_query_seconds": 66.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 86.042, "latency": 213.536, "stderr": 1.111, "cost_per_test": 0.04516, "avg_input_tokens": 138.4, "avg_output_tokens": 18793.5, "avg_reasoning_tokens": 17887.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 213.6, "median_query_seconds": 184.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 86.042, "latency": 244.712, "stderr": 1.133, "cost_per_test": 0.011043, "avg_input_tokens": 197.4, "avg_output_tokens": 37131.6, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 244.7, "median_query_seconds": 201.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 85.833, "latency": 143.907, "stderr": 1.318, "cost_per_test": 0.027139, "avg_input_tokens": 127.0, "avg_output_tokens": 2698.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 143.9, "median_query_seconds": 130.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 85.417, "latency": 924.913, "stderr": 1.327, "cost_per_test": 0.044508, "avg_input_tokens": 128.3, "avg_output_tokens": 20236.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 924.9, "median_query_seconds": 780.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 85.278, "latency": 266.179, "stderr": 1.389, "cost_per_test": 0.060099, "avg_input_tokens": 130.4, "avg_output_tokens": 7479.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 266.2, "median_query_seconds": 71.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 85.0, "latency": 102.246, "stderr": 0.534, "cost_per_test": 0.008281, "avg_input_tokens": 126.0, "avg_output_tokens": 16485.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 102.2, "median_query_seconds": 72.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 84.583, "latency": 373.072, "stderr": 1.125, "cost_per_test": 0.00442, "avg_input_tokens": 124.6, "avg_output_tokens": 11294.8, "avg_reasoning_tokens": 10914.0, "avg_cache_read_tokens": 82.3, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 373.0, "median_query_seconds": 301.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 83.958, "latency": 242.174, "stderr": 0.768, "cost_per_test": 0.011598, "avg_input_tokens": 136.2, "avg_output_tokens": 13145.2, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 242.2, "median_query_seconds": 204.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 83.667, "latency": 55.109, "stderr": 1.178, "cost_per_test": 0.025707, "avg_input_tokens": 130.4, "avg_output_tokens": 5810.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 55.1, "median_query_seconds": 37.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 83.542, "latency": 401.007, "stderr": 1.018, "cost_per_test": 0.086908, "avg_input_tokens": 127.2, "avg_output_tokens": 18292.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 401.0, "median_query_seconds": 323.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 83.333, "latency": 40.564, "stderr": 1.178, "cost_per_test": 0.006966, "avg_input_tokens": 128.0, "avg_output_tokens": 17382.8, "avg_reasoning_tokens": 16658.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 40.6, "median_query_seconds": 38.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 82.708, "latency": 158.25, "stderr": 0.915, "cost_per_test": 0.173775, "avg_input_tokens": 173.6, "avg_output_tokens": 26425.4, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 158.2, "median_query_seconds": 130.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 81.181, "latency": 193.682, "stderr": 1.29, "cost_per_test": 0.008862, "avg_input_tokens": 402.4, "avg_output_tokens": 22104.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 193.7, "median_query_seconds": 79.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 81.042, "latency": 16.76, "stderr": 1.1, "cost_per_test": 0.033364, "avg_input_tokens": 136.2, "avg_output_tokens": 5533.5, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 80.679, "latency": 267.982, "stderr": 1.208, "cost_per_test": 0.025112, "avg_input_tokens": 126.9, "avg_output_tokens": 16960.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 268.0, "median_query_seconds": 206.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 78.179, "latency": 214.504, "stderr": 1.629, "cost_per_test": 0.85994, "avg_input_tokens": 173.6, "avg_output_tokens": 11431.2, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 214.5, "median_query_seconds": 193.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 77.917, "latency": 237.635, "stderr": 1.289, "cost_per_test": 0.024152, "avg_input_tokens": 157.0, "avg_output_tokens": 20112.8, "avg_cache_read_tokens": 123.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 237.6, "median_query_seconds": 187.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 76.875, "latency": 17.442, "stderr": 1.389, "cost_per_test": 0.089253, "avg_input_tokens": 145.7, "avg_output_tokens": 1160.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 17.4, "median_query_seconds": 18.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 76.25, "latency": 271.788, "stderr": 1.435, "cost_per_test": 0.295026, "avg_input_tokens": 173.7, "avg_output_tokens": 19633.6, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 271.8, "median_query_seconds": 239.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 73.958, "latency": 153.912, "stderr": 1.343, "cost_per_test": 0.053393, "avg_input_tokens": 123.8, "avg_output_tokens": 9856.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 153.9, "median_query_seconds": 119.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 71.458, "latency": 177.028, "stderr": 1.458, "cost_per_test": 0.658977, "avg_input_tokens": 130.4, "avg_output_tokens": 10950.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 177.0, "median_query_seconds": 92.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 70.625, "latency": 31.397, "stderr": 0.562, "cost_per_test": 0.00293, "avg_input_tokens": 125.9, "avg_output_tokens": 5785.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 31.4, "median_query_seconds": 30.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 64.792, "latency": 119.555, "stderr": 2.046, "cost_per_test": 0.001399, "avg_input_tokens": 124.8, "avg_output_tokens": 3297.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 71.4, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 119.6, "median_query_seconds": 119.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 62.708, "latency": 124.668, "stderr": 1.217, "cost_per_test": 0.01278, "avg_input_tokens": 138.3, "avg_output_tokens": 4214.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 124.7, "median_query_seconds": 106.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 60.694, "latency": 206.753, "stderr": 2.562, "cost_per_test": 0.023054, "avg_input_tokens": 136.2, "avg_output_tokens": 3815.2, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 206.8, "median_query_seconds": 201.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-3": { "accuracy": 58.75, "latency": 63.989, "stderr": 1.439, "cost_per_test": 0.090668, "avg_input_tokens": 124.9, "avg_output_tokens": 6019.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 64.0, "median_query_seconds": 62.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 53.542, "latency": 167.893, "stderr": 1.596, "cost_per_test": 0.0, "avg_input_tokens": 152.4, "avg_output_tokens": 10716.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 167.9, "median_query_seconds": 163.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 52.202, "latency": 50.575, "stderr": 1.712, "cost_per_test": 0.003192, "avg_input_tokens": 123.8, "avg_output_tokens": 3423.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 50.6, "median_query_seconds": 48.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 51.458, "latency": 61.601, "stderr": 1.94, "cost_per_test": 0.004986, "avg_input_tokens": 127.0, "avg_output_tokens": 16772.9, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 61.6, "median_query_seconds": 51.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 49.792, "latency": 58.378, "stderr": 2.215, "cost_per_test": 0.004524, "avg_input_tokens": 127.0, "avg_output_tokens": 17404.9, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 58.4, "median_query_seconds": 44.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 49.375, "latency": 33.139, "stderr": 1.122, "cost_per_test": 0.0079, "avg_input_tokens": 131.4, "avg_output_tokens": 4905.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 33.2, "median_query_seconds": 20.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 44.583, "latency": 303.709, "stderr": 1.747, "cost_per_test": 0.395347, "avg_input_tokens": 173.7, "avg_output_tokens": 26321.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 303.8, "median_query_seconds": 300.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 44.236, "latency": 29.675, "stderr": 1.677, "cost_per_test": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 42.917, "latency": 104.325, "stderr": 0.884, "cost_per_test": 0.006086, "avg_input_tokens": 127.6, "avg_output_tokens": 4089.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 104.3, "median_query_seconds": 94.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 42.292, "latency": 65.954, "stderr": 1.334, "cost_per_test": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 42.083, "latency": 51.146, "stderr": 1.567, "cost_per_test": 0.000566, "avg_input_tokens": 127.0, "avg_output_tokens": 19973.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 51.2, "median_query_seconds": 39.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 41.25, "latency": 37.029, "stderr": 1.152, "cost_per_test": 0.110384, "avg_input_tokens": 145.7, "avg_output_tokens": 1442.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 37.0, "median_query_seconds": 33.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 39.583, "latency": 161.075, "stderr": 1.386, "cost_per_test": 0.050106, "avg_input_tokens": 131.4, "avg_output_tokens": 6230.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 161.0, "median_query_seconds": 25.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 38.542, "latency": 23.398, "stderr": 0.968, "cost_per_test": 0.02164, "avg_input_tokens": 145.7, "avg_output_tokens": 1413.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 23.4, "median_query_seconds": 22.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 33.333, "latency": 5.879, "stderr": 1.068, "cost_per_test": 0.000805, "avg_input_tokens": 245.9, "avg_output_tokens": 1512.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 5.8, "median_query_seconds": 3.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 29.792, "latency": 11.214, "stderr": 0.644, "cost_per_test": 0.001044, "avg_input_tokens": 127.0, "avg_output_tokens": 2577.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 11.2, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 27.5, "latency": 58.801, "stderr": 0.73, "cost_per_test": 0.003364, "avg_input_tokens": 123.8, "avg_output_tokens": 3614.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 58.8, "median_query_seconds": 43.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 27.083, "latency": 17.366, "stderr": 1.935, "cost_per_test": 0.000788, "avg_input_tokens": 286.9, "avg_output_tokens": 1539.5, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 256.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 17.4, "median_query_seconds": 9.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 26.458, "latency": 11.914, "stderr": 0.814, "cost_per_test": 0.001184, "avg_input_tokens": 131.4, "avg_output_tokens": 2927.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 12.0, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 26.25, "latency": 18.704, "stderr": 1.989, "cost_per_test": 0.003224, "avg_input_tokens": 127.0, "avg_output_tokens": 8027.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 18.7, "median_query_seconds": 14.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 25.208, "latency": 15.502, "stderr": 0.784, "cost_per_test": 0.001142, "avg_input_tokens": 128.9, "avg_output_tokens": 1266.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 15.5, "median_query_seconds": 15.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 22.292, "latency": 18.928, "stderr": 1.352, "cost_per_test": 0.023201, "avg_input_tokens": 145.7, "avg_output_tokens": 1517.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 18.9, "median_query_seconds": 14.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 18.958, "latency": 21.69, "stderr": 0.7, "cost_per_test": 0.000813, "avg_input_tokens": 128.9, "avg_output_tokens": 1338.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 21.7, "median_query_seconds": 20.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 18.75, "latency": 10.642, "stderr": 0.534, "cost_per_test": 0.004203, "avg_input_tokens": 127.0, "avg_output_tokens": 808.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 10.6, "median_query_seconds": 10.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 17.292, "latency": 5.696, "stderr": 0.901, "cost_per_test": 0.000266, "avg_input_tokens": 127.0, "avg_output_tokens": 853.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 16.042, "latency": 11.244, "stderr": 1.077, "cost_per_test": 0.001136, "avg_input_tokens": 159.4, "avg_output_tokens": 1131.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 11.2, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-2-1212": { "accuracy": 15.208, "latency": 57.884, "stderr": 0.784, "cost_per_test": 0.015018, "avg_input_tokens": 128.0, "avg_output_tokens": 1476.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 57.8, "median_query_seconds": 36.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 13.958, "latency": 68.368, "stderr": 0.968, "cost_per_test": 0.012158, "avg_input_tokens": 131.4, "avg_output_tokens": 1183.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 68.4, "median_query_seconds": 17.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 13.333, "latency": 23.345, "stderr": 0.945, "cost_per_test": 0.011732, "avg_input_tokens": 131.6, "avg_output_tokens": 1140.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 23.4, "median_query_seconds": 17.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 11.875, "latency": 15.776, "stderr": 0.955, "cost_per_test": 0.011954, "avg_input_tokens": 131.4, "avg_output_tokens": 1162.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 15.8, "median_query_seconds": 14.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 11.458, "latency": 28.771, "stderr": 1.03, "cost_per_test": 0.00101, "avg_input_tokens": 131.4, "avg_output_tokens": 1651.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 28.8, "median_query_seconds": 15.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 10.0, "latency": 9.188, "stderr": 0.938, "cost_per_test": 0.008821, "avg_input_tokens": 145.7, "avg_output_tokens": 559.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 9.2, "median_query_seconds": 9.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 9.375, "latency": 15.851, "stderr": 0.54, "cost_per_test": 0.0, "temperature": 0.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 9.167, "latency": 19.676, "stderr": 1.091, "cost_per_test": 0.006338, "avg_input_tokens": 137.9, "avg_output_tokens": 1010.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 19.6, "median_query_seconds": 14.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 5.625, "latency": 13.228, "stderr": 0.625, "cost_per_test": 0.000782, "avg_input_tokens": 128.0, "avg_output_tokens": 1261.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 13.2, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 3.542, "latency": 11.675, "stderr": 0.208, "cost_per_test": 0.000293, "avg_input_tokens": 128.0, "avg_output_tokens": 946.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 11.6, "median_query_seconds": 8.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 3.333, "latency": 9.055, "stderr": 0.315, "cost_per_test": 0.002432, "avg_input_tokens": 145.7, "avg_output_tokens": 578.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 9.1, "median_query_seconds": 9.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 0.417, "latency": 6.624, "stderr": 0.273, "cost_per_test": 0.000456, "avg_input_tokens": 144.8, "avg_output_tokens": 1066.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 6.6, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 0.417, "latency": 18.858, "stderr": 0.417, "cost_per_test": 0.007658, "avg_input_tokens": 144.8, "avg_output_tokens": 921.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 18.9, "median_query_seconds": 17.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "aime_2024": { "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 99.583, "latency": 196.019, "stderr": 0.417, "cost_per_test": 1.212538, "avg_input_tokens": 167.8, "avg_output_tokens": 16133.6, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 196.0, "median_query_seconds": 181.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 99.167, "latency": 93.324, "stderr": 0.546, "cost_per_test": 0.111396, "avg_input_tokens": 123.2, "avg_output_tokens": 9262.5, "avg_reasoning_tokens": 8360.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 93.3, "median_query_seconds": 62.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 99.167, "latency": 105.646, "stderr": 0.546, "cost_per_test": 0.162595, "avg_input_tokens": 123.2, "avg_output_tokens": 13529.0, "avg_reasoning_tokens": 12582.6, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 105.7, "median_query_seconds": 96.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 98.75, "latency": 97.839, "stderr": 0.61, "cost_per_test": 0.051914, "avg_input_tokens": 237.8, "avg_output_tokens": 8627.6, "avg_reasoning_tokens": 7766.8, "avg_cache_read_tokens": 181.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 97.8, "median_query_seconds": 89.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 97.5, "latency": 248.186, "stderr": 0.833, "cost_per_test": 0.285745, "avg_input_tokens": 155.8, "avg_output_tokens": 19018.5, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 248.2, "median_query_seconds": 91.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 97.5, "latency": 294.43, "stderr": 0.546, "cost_per_test": 0.058907, "avg_input_tokens": 123.3, "avg_output_tokens": 19628.6, "avg_cache_read_tokens": 106.2, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 294.4, "median_query_seconds": 240.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 97.083, "latency": 101.103, "stderr": 0.417, "cost_per_test": 0.120107, "avg_input_tokens": 127.5, "avg_output_tokens": 8563.1, "avg_reasoning_tokens": 7886.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 101.1, "median_query_seconds": 44.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 97.083, "latency": 102.378, "stderr": 0.984, "cost_per_test": 0.117288, "avg_input_tokens": 125.1, "avg_output_tokens": 7798.3, "avg_reasoning_tokens": 7186.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 102.4, "median_query_seconds": 57.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 96.667, "latency": 127.304, "stderr": 0.63, "cost_per_test": 0.003681, "avg_input_tokens": 175.6, "avg_output_tokens": 6091.7, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 127.3, "median_query_seconds": 90.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 96.667, "latency": 136.711, "stderr": 0.0, "cost_per_test": 0.339778, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 95.833, "latency": 80.8, "stderr": 0.833, "cost_per_test": 0.04424, "avg_input_tokens": 123.2, "avg_output_tokens": 14726.3, "avg_reasoning_tokens": 13917.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 80.8, "median_query_seconds": 81.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 95.833, "latency": 290.099, "stderr": 0.546, "cost_per_test": 0.118454, "avg_input_tokens": 125.1, "avg_output_tokens": 26302.2, "avg_reasoning_tokens": 25858.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 290.1, "median_query_seconds": 241.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 95.417, "latency": 510.364, "stderr": 1.08, "cost_per_test": 0.092649, "avg_input_tokens": 133.3, "avg_output_tokens": 30860.7, "avg_reasoning_tokens": 30034.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 510.4, "median_query_seconds": 430.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-4.7": { "accuracy": 95.0, "latency": 502.917, "stderr": 0.63, "cost_per_test": 0.038545, "avg_input_tokens": 124.4, "avg_output_tokens": 17562.3, "avg_cache_read_tokens": 19.9, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 502.9, "median_query_seconds": 331.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-4.6": { "accuracy": 94.583, "latency": 625.269, "stderr": 0.877, "cost_per_test": 0.102339, "avg_input_tokens": 126.2, "avg_output_tokens": 13105.6, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 625.3, "median_query_seconds": 183.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 94.167, "latency": 31.664, "stderr": 0.833, "cost_per_test": 0.003984, "avg_input_tokens": 230.8, "avg_output_tokens": 7875.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 31.6, "median_query_seconds": 19.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 94.167, "latency": 336.233, "stderr": 0.833, "cost_per_test": 0.085431, "avg_input_tokens": 943.7, "avg_output_tokens": 8542.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 336.2, "median_query_seconds": 93.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 93.75, "latency": 145.315, "stderr": 0.984, "cost_per_test": 0.132848, "avg_input_tokens": 125.1, "avg_output_tokens": 13269.1, "avg_reasoning_tokens": 11809.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 145.3, "median_query_seconds": 84.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 93.333, "latency": 83.322, "stderr": 0.891, "cost_per_test": 0.00373, "avg_input_tokens": 269.8, "avg_output_tokens": 7424.1, "avg_reasoning_tokens": 6863.5, "avg_cache_read_tokens": 239.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 83.3, "median_query_seconds": 55.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-0709": { "accuracy": 93.333, "latency": 103.985, "stderr": 3.333, "cost_per_test": 0.141831, "avg_input_tokens": 120.8, "avg_output_tokens": 9431.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 104.0, "median_query_seconds": 82.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 93.333, "latency": 230.487, "stderr": 1.409, "cost_per_test": 0.010694, "avg_input_tokens": 133.3, "avg_output_tokens": 26701.9, "avg_reasoning_tokens": 25424.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 230.5, "median_query_seconds": 166.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 93.333, "latency": 267.082, "stderr": 0.0, "cost_per_test": 0.351439, "avg_input_tokens": 155.8, "avg_output_tokens": 23398.1, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 267.1, "median_query_seconds": 142.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-5-thinking": { "accuracy": 93.333, "latency": 410.326, "stderr": 1.091, "cost_per_test": 0.062435, "avg_input_tokens": 124.1, "avg_output_tokens": 19621.1, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 410.3, "median_query_seconds": 276.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 93.113, "latency": 162.754, "stderr": 1.908, "cost_per_test": 0.00998, "avg_input_tokens": 193.8, "avg_output_tokens": 16585.5, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.1, "avg_query_seconds": 162.8, "median_query_seconds": 127.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 92.917, "latency": 86.475, "stderr": 0.755, "cost_per_test": 0.020259, "avg_input_tokens": 125.1, "avg_output_tokens": 10113.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 86.5, "median_query_seconds": 41.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 92.083, "latency": 339.408, "stderr": 0.61, "cost_per_test": 0.063791, "avg_input_tokens": 124.3, "avg_output_tokens": 19905.6, "avg_reasoning_tokens": 19427.1, "avg_cache_read_tokens": 39.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 339.4, "median_query_seconds": 254.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 91.667, "latency": 236.514, "stderr": 1.26, "cost_per_test": 0.022761, "avg_input_tokens": 134.7, "avg_output_tokens": 18955.6, "avg_cache_read_tokens": 105.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 236.5, "median_query_seconds": 141.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 90.667, "latency": 205.318, "stderr": 0.667, "cost_per_test": 0.249445, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 90.417, "latency": 17.271, "stderr": 2.13, "cost_per_test": 0.088484, "avg_input_tokens": 138.8, "avg_output_tokens": 1152.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 17.3, "median_query_seconds": 17.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 89.167, "latency": 175.552, "stderr": 1.045, "cost_per_test": 0.049871, "avg_input_tokens": 125.1, "avg_output_tokens": 11303.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 175.6, "median_query_seconds": 56.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 89.167, "latency": 343.683, "stderr": 1.373, "cost_per_test": 0.038321, "avg_input_tokens": 154.7, "avg_output_tokens": 15952.7, "avg_cache_read_tokens": 67.4, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 343.7, "median_query_seconds": 189.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 88.75, "latency": 94.102, "stderr": 0.877, "cost_per_test": 0.007605, "avg_input_tokens": 120.8, "avg_output_tokens": 15136.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 94.1, "median_query_seconds": 70.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 88.333, "latency": 38.16, "stderr": 0.891, "cost_per_test": 0.00649, "avg_input_tokens": 123.2, "avg_output_tokens": 16193.8, "avg_reasoning_tokens": 15481.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 38.2, "median_query_seconds": 35.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 88.333, "latency": 202.315, "stderr": 1.543, "cost_per_test": 0.042889, "avg_input_tokens": 133.3, "avg_output_tokens": 17848.1, "avg_reasoning_tokens": 16969.7, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 202.3, "median_query_seconds": 163.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 87.5, "latency": 121.572, "stderr": 1.596, "cost_per_test": 0.025552, "avg_input_tokens": 122.2, "avg_output_tokens": 2540.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 121.6, "median_query_seconds": 99.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 87.5, "latency": 356.683, "stderr": 1.045, "cost_per_test": 0.004382, "avg_input_tokens": 119.1, "avg_output_tokens": 10657.0, "avg_reasoning_tokens": 10279.4, "avg_cache_read_tokens": 77.3, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 356.7, "median_query_seconds": 269.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 87.222, "latency": 173.738, "stderr": 2.344, "cost_per_test": 0.052328, "avg_input_tokens": 125.1, "avg_output_tokens": 6509.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 173.7, "median_query_seconds": 59.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.5": { "accuracy": 87.083, "latency": 322.751, "stderr": 1.327, "cost_per_test": 0.044344, "avg_input_tokens": 126.3, "avg_output_tokens": 20122.2, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 322.8, "median_query_seconds": 250.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 87.083, "latency": 374.391, "stderr": 1.598, "cost_per_test": 0.0798, "avg_input_tokens": 121.3, "avg_output_tokens": 16929.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 374.4, "median_query_seconds": 332.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 86.667, "latency": 236.967, "stderr": 1.409, "cost_per_test": 0.010258, "avg_input_tokens": 191.8, "avg_output_tokens": 34542.6, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 237.0, "median_query_seconds": 192.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 85.833, "latency": 15.159, "stderr": 1.22, "cost_per_test": 0.031117, "avg_input_tokens": 131.1, "avg_output_tokens": 5160.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 85.833, "latency": 809.713, "stderr": 2.065, "cost_per_test": 0.038294, "avg_input_tokens": 125.2, "avg_output_tokens": 17522.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 809.7, "median_query_seconds": 649.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 85.417, "latency": 58.769, "stderr": 1.4, "cost_per_test": 0.002955, "avg_input_tokens": 129.5, "avg_output_tokens": 7371.7, "avg_reasoning_tokens": 6573.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 58.8, "median_query_seconds": 23.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 84.667, "latency": 50.336, "stderr": 2.0, "cost_per_test": 0.022297, "avg_input_tokens": 125.1, "avg_output_tokens": 5036.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 50.3, "median_query_seconds": 30.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 84.583, "latency": 227.556, "stderr": 0.61, "cost_per_test": 0.010654, "avg_input_tokens": 131.1, "avg_output_tokens": 12073.7, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 227.6, "median_query_seconds": 190.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 84.583, "latency": 262.151, "stderr": 1.08, "cost_per_test": 0.02346, "avg_input_tokens": 120.8, "avg_output_tokens": 15731.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 262.2, "median_query_seconds": 199.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 83.75, "latency": 152.656, "stderr": 0.755, "cost_per_test": 0.174909, "avg_input_tokens": 166.8, "avg_output_tokens": 25281.2, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 152.6, "median_query_seconds": 116.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 83.441, "latency": 211.203, "stderr": 1.45, "cost_per_test": 0.845636, "avg_input_tokens": 166.4, "avg_output_tokens": 11241.9, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 211.2, "median_query_seconds": 172.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 82.5, "latency": 264.28, "stderr": 1.637, "cost_per_test": 0.283075, "avg_input_tokens": 166.8, "avg_output_tokens": 18838.3, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 264.3, "median_query_seconds": 231.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 81.111, "latency": 193.162, "stderr": 1.648, "cost_per_test": 0.008014, "avg_input_tokens": 125.1, "avg_output_tokens": 20019.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 193.2, "median_query_seconds": 66.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 79.167, "latency": 217.936, "stderr": 1.754, "cost_per_test": 0.022637, "avg_input_tokens": 151.7, "avg_output_tokens": 18851.3, "avg_cache_read_tokens": 119.6, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 217.9, "median_query_seconds": 171.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 77.5, "latency": 153.052, "stderr": 1.51, "cost_per_test": 0.048844, "avg_input_tokens": 118.2, "avg_output_tokens": 9015.7, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 153.0, "median_query_seconds": 101.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 73.75, "latency": 30.443, "stderr": 0.755, "cost_per_test": 0.002841, "avg_input_tokens": 120.8, "avg_output_tokens": 5609.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 30.4, "median_query_seconds": 30.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 73.333, "latency": 166.783, "stderr": 1.992, "cost_per_test": 0.634309, "avg_input_tokens": 125.1, "avg_output_tokens": 10540.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 166.8, "median_query_seconds": 84.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 70.0, "latency": 110.392, "stderr": 1.543, "cost_per_test": 0.011401, "avg_input_tokens": 133.0, "avg_output_tokens": 3756.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 110.4, "median_query_seconds": 91.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 69.167, "latency": 120.589, "stderr": 2.938, "cost_per_test": 0.00137, "avg_input_tokens": 119.2, "avg_output_tokens": 3221.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 66.1, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 120.6, "median_query_seconds": 116.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 65.417, "latency": 182.155, "stderr": 3.208, "cost_per_test": 0.022332, "avg_input_tokens": 131.1, "avg_output_tokens": 3695.7, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 182.2, "median_query_seconds": 186.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 61.25, "latency": 161.92, "stderr": 2.813, "cost_per_test": 0.0, "avg_input_tokens": 147.1, "avg_output_tokens": 10383.7, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 161.9, "median_query_seconds": 154.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-3": { "accuracy": 60.417, "latency": 64.248, "stderr": 2.394, "cost_per_test": 0.092816, "avg_input_tokens": 119.8, "avg_output_tokens": 6163.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 64.2, "median_query_seconds": 61.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 58.571, "latency": 49.485, "stderr": 2.804, "cost_per_test": 0.003187, "avg_input_tokens": 118.2, "avg_output_tokens": 3422.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 49.5, "median_query_seconds": 46.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 54.583, "latency": 57.704, "stderr": 2.741, "cost_per_test": 0.005092, "avg_input_tokens": 122.2, "avg_output_tokens": 14899.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 57.7, "median_query_seconds": 45.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 54.583, "latency": 101.853, "stderr": 0.61, "cost_per_test": 0.005864, "avg_input_tokens": 121.6, "avg_output_tokens": 3964.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 101.8, "median_query_seconds": 91.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 52.917, "latency": 30.512, "stderr": 1.83, "cost_per_test": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 52.917, "latency": 55.114, "stderr": 2.917, "cost_per_test": 0.004435, "avg_input_tokens": 122.2, "avg_output_tokens": 15967.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 55.1, "median_query_seconds": 42.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 52.5, "latency": 44.809, "stderr": 2.578, "cost_per_test": 0.000551, "avg_input_tokens": 122.2, "avg_output_tokens": 17305.4, "avg_reasoning_tokens": 15958.1, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 44.8, "median_query_seconds": 33.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 50.833, "latency": 33.709, "stderr": 1.51, "cost_per_test": 0.008168, "avg_input_tokens": 126.1, "avg_output_tokens": 5073.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 33.7, "median_query_seconds": 20.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 50.0, "latency": 68.044, "stderr": 1.89, "cost_per_test": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 49.167, "latency": 293.56, "stderr": 2.159, "cost_per_test": 0.380382, "avg_input_tokens": 166.8, "avg_output_tokens": 25325.5, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 293.6, "median_query_seconds": 290.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 48.75, "latency": 38.331, "stderr": 1.4, "cost_per_test": 0.11308, "avg_input_tokens": 138.8, "avg_output_tokens": 1480.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 38.3, "median_query_seconds": 33.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 48.75, "latency": 245.542, "stderr": 2.435, "cost_per_test": 0.04257, "avg_input_tokens": 126.1, "avg_output_tokens": 5289.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 245.5, "median_query_seconds": 27.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 43.333, "latency": 23.206, "stderr": 1.667, "cost_per_test": 0.021848, "avg_input_tokens": 138.8, "avg_output_tokens": 1428.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 23.2, "median_query_seconds": 22.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 32.917, "latency": 15.429, "stderr": 1.327, "cost_per_test": 0.001131, "avg_input_tokens": 123.6, "avg_output_tokens": 1254.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 15.4, "median_query_seconds": 15.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 32.5, "latency": 12.103, "stderr": 0.833, "cost_per_test": 0.001136, "avg_input_tokens": 122.2, "avg_output_tokens": 2811.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 12.1, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 31.25, "latency": 17.094, "stderr": 2.176, "cost_per_test": 0.002931, "avg_input_tokens": 122.2, "avg_output_tokens": 7297.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 17.1, "median_query_seconds": 12.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 30.417, "latency": 66.28, "stderr": 1.327, "cost_per_test": 0.003775, "avg_input_tokens": 118.2, "avg_output_tokens": 4076.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 66.3, "median_query_seconds": 58.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 28.75, "latency": 13.068, "stderr": 1.25, "cost_per_test": 0.001266, "avg_input_tokens": 126.1, "avg_output_tokens": 3132.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 13.1, "median_query_seconds": 11.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 27.917, "latency": 21.568, "stderr": 1.08, "cost_per_test": 0.000789, "avg_input_tokens": 123.6, "avg_output_tokens": 1299.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 21.6, "median_query_seconds": 19.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 26.25, "latency": 19.427, "stderr": 1.327, "cost_per_test": 0.023944, "avg_input_tokens": 138.8, "avg_output_tokens": 1568.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 19.4, "median_query_seconds": 14.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 24.167, "latency": 11.446, "stderr": 1.373, "cost_per_test": 0.001148, "avg_input_tokens": 154.1, "avg_output_tokens": 1150.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 11.4, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 23.75, "latency": 10.658, "stderr": 0.755, "cost_per_test": 0.00421, "avg_input_tokens": 122.2, "avg_output_tokens": 811.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 10.6, "median_query_seconds": 10.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 20.833, "latency": 6.92, "stderr": 0.833, "cost_per_test": 0.00097, "avg_input_tokens": 240.8, "avg_output_tokens": 1844.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 6.9, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 20.833, "latency": 12.902, "stderr": 2.8, "cost_per_test": 0.00059, "avg_input_tokens": 281.8, "avg_output_tokens": 1144.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 254.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 12.9, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-2-1212": { "accuracy": 19.583, "latency": 58.028, "stderr": 1.327, "cost_per_test": 0.014342, "avg_input_tokens": 122.7, "avg_output_tokens": 1409.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 58.0, "median_query_seconds": 39.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 17.083, "latency": 5.826, "stderr": 1.327, "cost_per_test": 0.000271, "avg_input_tokens": 122.2, "avg_output_tokens": 872.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 5.8, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 15.417, "latency": 8.98, "stderr": 1.535, "cost_per_test": 0.008645, "avg_input_tokens": 138.8, "avg_output_tokens": 548.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 9.0, "median_query_seconds": 8.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 15.0, "latency": 23.494, "stderr": 1.543, "cost_per_test": 0.012986, "avg_input_tokens": 126.1, "avg_output_tokens": 1267.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 23.5, "median_query_seconds": 18.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 14.167, "latency": 15.959, "stderr": 1.045, "cost_per_test": 0.011915, "avg_input_tokens": 126.1, "avg_output_tokens": 1160.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 16.0, "median_query_seconds": 13.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 13.333, "latency": 16.094, "stderr": 0.63, "cost_per_test": 0.0, "temperature": 0.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 12.917, "latency": 101.51, "stderr": 1.469, "cost_per_test": 0.014132, "avg_input_tokens": 126.1, "avg_output_tokens": 1381.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 101.5, "median_query_seconds": 16.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 11.667, "latency": 21.843, "stderr": 0.891, "cost_per_test": 0.006976, "avg_input_tokens": 133.1, "avg_output_tokens": 1118.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 21.8, "median_query_seconds": 14.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 8.75, "latency": 40.44, "stderr": 1.535, "cost_per_test": 0.001342, "avg_input_tokens": 126.1, "avg_output_tokens": 2206.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 40.4, "median_query_seconds": 15.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 7.917, "latency": 17.568, "stderr": 0.61, "cost_per_test": 0.001034, "avg_input_tokens": 122.4, "avg_output_tokens": 1683.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 17.6, "median_query_seconds": 8.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 6.667, "latency": 8.758, "stderr": 0.63, "cost_per_test": 0.002344, "avg_input_tokens": 138.8, "avg_output_tokens": 558.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 8.8, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 3.75, "latency": 13.125, "stderr": 0.417, "cost_per_test": 0.00034, "avg_input_tokens": 122.4, "avg_output_tokens": 1104.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 13.1, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 0.833, "latency": 6.411, "stderr": 0.546, "cost_per_test": 0.000438, "avg_input_tokens": 139.7, "avg_output_tokens": 1025.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 6.4, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 0.833, "latency": 18.75, "stderr": 0.833, "cost_per_test": 0.007623, "avg_input_tokens": 139.7, "avg_output_tokens": 917.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 18.8, "median_query_seconds": 16.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "aime_2025": { "google/gemini-3.1-pro-preview": { "accuracy": 97.083, "latency": 115.726, "stderr": 0.417, "cost_per_test": 0.142586, "avg_input_tokens": 132.8, "avg_output_tokens": 11860.1, "avg_reasoning_tokens": 10852.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 115.7, "median_query_seconds": 90.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark": { "accuracy": 97.083, "latency": 186.187, "stderr": 0.417, "cost_per_test": 0.005441, "avg_input_tokens": 186.2, "avg_output_tokens": 9021.4, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 186.2, "median_query_seconds": 133.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 96.667, "latency": 139.517, "stderr": 0.0, "cost_per_test": 0.150539, "avg_input_tokens": 592.0, "avg_output_tokens": 14432.9, "avg_reasoning_tokens": 10345.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 29, "avg_turns": 1.0, "avg_query_seconds": 139.5, "median_query_seconds": 68.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 96.25, "latency": 140.554, "stderr": 0.417, "cost_per_test": 0.159085, "avg_input_tokens": 135.6, "avg_output_tokens": 10583.0, "avg_reasoning_tokens": 9887.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 140.6, "median_query_seconds": 74.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 95.833, "latency": 227.418, "stderr": 1.045, "cost_per_test": 0.561248, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 95.417, "latency": 87.215, "stderr": 0.61, "cost_per_test": 0.048415, "avg_input_tokens": 132.8, "avg_output_tokens": 16116.2, "avg_reasoning_tokens": 15274.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 87.2, "median_query_seconds": 94.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 95.417, "latency": 310.003, "stderr": 0.61, "cost_per_test": 0.138819, "avg_input_tokens": 179.1, "avg_output_tokens": 30818.8, "avg_reasoning_tokens": 30321.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 310.0, "median_query_seconds": 267.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 94.194, "latency": 112.882, "stderr": 0.54, "cost_per_test": 0.176016, "avg_input_tokens": 158.1, "avg_output_tokens": 14641.7, "avg_reasoning_tokens": 13684.2, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 112.9, "median_query_seconds": 111.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 94.167, "latency": 115.759, "stderr": 0.833, "cost_per_test": 0.06688, "avg_input_tokens": 250.1, "avg_output_tokens": 11122.8, "avg_reasoning_tokens": 10310.3, "avg_cache_read_tokens": 198.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 115.8, "median_query_seconds": 97.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 93.75, "latency": 343.169, "stderr": 0.984, "cost_per_test": 0.401897, "avg_input_tokens": 169.6, "avg_output_tokens": 26759.2, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 343.2, "median_query_seconds": 203.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 93.75, "latency": 350.791, "stderr": 0.417, "cost_per_test": 0.069378, "avg_input_tokens": 133.8, "avg_output_tokens": 23118.9, "avg_cache_read_tokens": 117.1, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 350.8, "median_query_seconds": 301.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 93.75, "latency": 522.617, "stderr": 0.417, "cost_per_test": 0.09439, "avg_input_tokens": 143.5, "avg_output_tokens": 31439.4, "avg_reasoning_tokens": 30579.3, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 522.6, "median_query_seconds": 479.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 92.917, "latency": 182.468, "stderr": 0.755, "cost_per_test": 0.16786, "avg_input_tokens": 135.8, "avg_output_tokens": 16769.1, "avg_reasoning_tokens": 15127.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 182.5, "median_query_seconds": 112.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 92.581, "latency": 248.171, "stderr": 2.474, "cost_per_test": 0.099955, "avg_input_tokens": 846.9, "avg_output_tokens": 10102.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 248.2, "median_query_seconds": 128.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 92.083, "latency": 72.984, "stderr": 1.4, "cost_per_test": 0.003908, "avg_input_tokens": 428.6, "avg_output_tokens": 9716.5, "avg_reasoning_tokens": 8813.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 73.0, "median_query_seconds": 31.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 92.083, "latency": 150.504, "stderr": 1.535, "cost_per_test": 0.010965, "avg_input_tokens": 203.0, "avg_output_tokens": 18522.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 150.5, "median_query_seconds": 95.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 91.667, "latency": 248.72, "stderr": 1.091, "cost_per_test": 0.01154, "avg_input_tokens": 143.5, "avg_output_tokens": 28815.1, "avg_reasoning_tokens": 27449.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 248.7, "median_query_seconds": 197.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.1": { "accuracy": 91.667, "latency": 410.994, "stderr": 0.63, "cost_per_test": 0.078105, "avg_input_tokens": 135.0, "avg_output_tokens": 24375.0, "avg_reasoning_tokens": 23862.4, "avg_cache_read_tokens": 37.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 411.0, "median_query_seconds": 360.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-4.7": { "accuracy": 91.667, "latency": 555.468, "stderr": 0.63, "cost_per_test": 0.048071, "avg_input_tokens": 135.0, "avg_output_tokens": 21819.5, "avg_cache_read_tokens": 25.6, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 555.5, "median_query_seconds": 406.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 91.25, "latency": 228.383, "stderr": 0.61, "cost_per_test": 1.427953, "avg_input_tokens": 181.6, "avg_output_tokens": 19003.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 228.4, "median_query_seconds": 218.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 91.25, "latency": 388.987, "stderr": 0.877, "cost_per_test": 0.520585, "avg_input_tokens": 169.6, "avg_output_tokens": 34671.7, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 389.0, "median_query_seconds": 280.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.6": { "accuracy": 90.833, "latency": 774.468, "stderr": 0.833, "cost_per_test": 0.130232, "avg_input_tokens": 136.5, "avg_output_tokens": 20129.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 774.4, "median_query_seconds": 233.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 90.417, "latency": 100.673, "stderr": 0.755, "cost_per_test": 0.004458, "avg_input_tokens": 280.0, "avg_output_tokens": 8877.1, "avg_reasoning_tokens": 8329.7, "avg_cache_read_tokens": 241.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 100.7, "median_query_seconds": 57.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 90.417, "latency": 236.759, "stderr": 0.755, "cost_per_test": 0.023157, "avg_input_tokens": 145.3, "avg_output_tokens": 19287.9, "avg_cache_read_tokens": 125.7, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 236.8, "median_query_seconds": 153.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 90.0, "latency": 124.585, "stderr": 1.491, "cost_per_test": 0.025261, "avg_input_tokens": 135.8, "avg_output_tokens": 12613.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 124.6, "median_query_seconds": 40.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 90.0, "latency": 507.062, "stderr": 1.091, "cost_per_test": 0.077199, "avg_input_tokens": 135.0, "avg_output_tokens": 24084.2, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 507.1, "median_query_seconds": 426.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 88.333, "latency": 37.454, "stderr": 1.89, "cost_per_test": 0.00465, "avg_input_tokens": 241.0, "avg_output_tokens": 9204.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 37.4, "median_query_seconds": 18.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 88.333, "latency": 403.292, "stderr": 1.091, "cost_per_test": 0.044437, "avg_input_tokens": 165.3, "avg_output_tokens": 18499.4, "avg_cache_read_tokens": 68.2, "avg_cache_write_tokens": 97.1, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 403.3, "median_query_seconds": 265.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4-0709": { "accuracy": 87.778, "latency": 162.471, "stderr": 1.111, "cost_per_test": 0.206967, "avg_input_tokens": 131.0, "avg_output_tokens": 13771.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 162.5, "median_query_seconds": 76.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-4.5": { "accuracy": 86.25, "latency": 343.484, "stderr": 2.309, "cost_per_test": 0.049437, "avg_input_tokens": 137.0, "avg_output_tokens": 22433.9, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 343.5, "median_query_seconds": 302.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 85.714, "latency": 229.69, "stderr": 1.198, "cost_per_test": 0.284409, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 85.417, "latency": 252.457, "stderr": 1.775, "cost_per_test": 0.011828, "avg_input_tokens": 203.1, "avg_output_tokens": 39720.7, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 252.4, "median_query_seconds": 211.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 85.0, "latency": 1040.114, "stderr": 1.667, "cost_per_test": 0.050722, "avg_input_tokens": 131.4, "avg_output_tokens": 22951.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 1040.1, "median_query_seconds": 911.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 84.167, "latency": 166.242, "stderr": 2.097, "cost_per_test": 0.028726, "avg_input_tokens": 131.9, "avg_output_tokens": 2856.1, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 166.2, "median_query_seconds": 160.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 83.75, "latency": 133.747, "stderr": 1.83, "cost_per_test": 0.051763, "avg_input_tokens": 135.8, "avg_output_tokens": 11730.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 133.7, "median_query_seconds": 76.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 83.75, "latency": 224.758, "stderr": 1.598, "cost_per_test": 0.047431, "avg_input_tokens": 143.5, "avg_output_tokens": 19738.9, "avg_reasoning_tokens": 18805.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 224.8, "median_query_seconds": 205.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 83.333, "latency": 256.792, "stderr": 1.409, "cost_per_test": 0.012542, "avg_input_tokens": 141.4, "avg_output_tokens": 14216.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 256.8, "median_query_seconds": 219.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 83.333, "latency": 358.62, "stderr": 1.491, "cost_per_test": 0.06787, "avg_input_tokens": 135.8, "avg_output_tokens": 8449.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 358.6, "median_query_seconds": 82.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 82.667, "latency": 59.883, "stderr": 1.247, "cost_per_test": 0.029117, "avg_input_tokens": 135.8, "avg_output_tokens": 6583.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 59.9, "median_query_seconds": 44.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 81.667, "latency": 163.843, "stderr": 1.667, "cost_per_test": 0.172641, "avg_input_tokens": 180.4, "avg_output_tokens": 27569.7, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 163.8, "median_query_seconds": 144.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 81.667, "latency": 389.462, "stderr": 1.992, "cost_per_test": 0.004458, "avg_input_tokens": 130.2, "avg_output_tokens": 11932.6, "avg_reasoning_tokens": 11548.6, "avg_cache_read_tokens": 87.3, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 389.4, "median_query_seconds": 334.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 81.25, "latency": 110.39, "stderr": 0.61, "cost_per_test": 0.008957, "avg_input_tokens": 131.3, "avg_output_tokens": 17834.9, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 110.4, "median_query_seconds": 73.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 81.25, "latency": 194.202, "stderr": 1.986, "cost_per_test": 0.00971, "avg_input_tokens": 679.8, "avg_output_tokens": 24188.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 194.2, "median_query_seconds": 92.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 80.0, "latency": 427.624, "stderr": 1.26, "cost_per_test": 0.094015, "avg_input_tokens": 133.2, "avg_output_tokens": 19656.5, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 427.6, "median_query_seconds": 315.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 78.333, "latency": 42.968, "stderr": 2.182, "cost_per_test": 0.007442, "avg_input_tokens": 132.8, "avg_output_tokens": 18571.8, "avg_reasoning_tokens": 17835.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 43.0, "median_query_seconds": 42.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 76.774, "latency": 273.812, "stderr": 2.161, "cost_per_test": 0.026765, "avg_input_tokens": 133.0, "avg_output_tokens": 18188.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 273.8, "median_query_seconds": 212.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 76.667, "latency": 257.333, "stderr": 1.89, "cost_per_test": 0.025666, "avg_input_tokens": 162.3, "avg_output_tokens": 21374.4, "avg_cache_read_tokens": 128.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 257.3, "median_query_seconds": 204.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3-max": { "accuracy": 76.25, "latency": 18.362, "stderr": 1.83, "cost_per_test": 0.035612, "avg_input_tokens": 141.3, "avg_output_tokens": 5907.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 72.917, "latency": 217.805, "stderr": 2.917, "cost_per_test": 0.874244, "avg_input_tokens": 180.9, "avg_output_tokens": 11620.4, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 217.8, "median_query_seconds": 213.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 70.417, "latency": 154.772, "stderr": 2.222, "cost_per_test": 0.057942, "avg_input_tokens": 129.3, "avg_output_tokens": 10697.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 154.8, "median_query_seconds": 136.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 70.0, "latency": 279.296, "stderr": 2.357, "cost_per_test": 0.306977, "avg_input_tokens": 180.6, "avg_output_tokens": 20429.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 279.3, "median_query_seconds": 247.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 69.583, "latency": 187.272, "stderr": 2.13, "cost_per_test": 0.683645, "avg_input_tokens": 135.8, "avg_output_tokens": 11360.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 187.3, "median_query_seconds": 100.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 67.5, "latency": 32.352, "stderr": 0.833, "cost_per_test": 0.00302, "avg_input_tokens": 131.0, "avg_output_tokens": 5961.1, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 32.4, "median_query_seconds": 30.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 63.333, "latency": 17.614, "stderr": 1.782, "cost_per_test": 0.090022, "avg_input_tokens": 152.6, "avg_output_tokens": 1169.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 17.6, "median_query_seconds": 18.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 60.417, "latency": 118.521, "stderr": 2.848, "cost_per_test": 0.001428, "avg_input_tokens": 130.3, "avg_output_tokens": 3372.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 76.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 118.5, "median_query_seconds": 122.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-3": { "accuracy": 57.083, "latency": 63.73, "stderr": 1.598, "cost_per_test": 0.088521, "avg_input_tokens": 130.0, "avg_output_tokens": 5875.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 63.7, "median_query_seconds": 63.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 55.972, "latency": 231.351, "stderr": 3.995, "cost_per_test": 0.023777, "avg_input_tokens": 141.2, "avg_output_tokens": 3934.6, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 231.4, "median_query_seconds": 216.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 55.417, "latency": 138.944, "stderr": 1.883, "cost_per_test": 0.01416, "avg_input_tokens": 143.6, "avg_output_tokens": 4672.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 139.0, "median_query_seconds": 120.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 48.333, "latency": 65.497, "stderr": 2.746, "cost_per_test": 0.004879, "avg_input_tokens": 131.9, "avg_output_tokens": 18646.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 65.5, "median_query_seconds": 58.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 47.917, "latency": 32.569, "stderr": 1.659, "cost_per_test": 0.007633, "avg_input_tokens": 136.8, "avg_output_tokens": 4736.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 32.6, "median_query_seconds": 20.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 46.667, "latency": 61.642, "stderr": 3.333, "cost_per_test": 0.004614, "avg_input_tokens": 131.9, "avg_output_tokens": 18842.8, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 61.6, "median_query_seconds": 47.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 45.833, "latency": 4.837, "stderr": 1.967, "cost_per_test": 0.00064, "avg_input_tokens": 251.0, "avg_output_tokens": 1180.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 4.8, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 45.833, "latency": 51.665, "stderr": 1.967, "cost_per_test": 0.003198, "avg_input_tokens": 129.3, "avg_output_tokens": 3424.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 51.7, "median_query_seconds": 49.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 45.833, "latency": 173.867, "stderr": 1.51, "cost_per_test": 0.0, "avg_input_tokens": 157.8, "avg_output_tokens": 11048.3, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 173.9, "median_query_seconds": 173.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 40.0, "latency": 313.858, "stderr": 2.746, "cost_per_test": 0.410312, "avg_input_tokens": 180.6, "avg_output_tokens": 27318.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 313.9, "median_query_seconds": 310.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 35.556, "latency": 28.838, "stderr": 2.811, "cost_per_test": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 34.583, "latency": 63.864, "stderr": 1.883, "cost_per_test": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 33.75, "latency": 23.59, "stderr": 0.984, "cost_per_test": 0.021431, "avg_input_tokens": 152.6, "avg_output_tokens": 1398.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 23.6, "median_query_seconds": 22.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 33.75, "latency": 35.727, "stderr": 1.83, "cost_per_test": 0.107687, "avg_input_tokens": 152.6, "avg_output_tokens": 1405.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 35.7, "median_query_seconds": 32.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 33.333, "latency": 21.828, "stderr": 2.673, "cost_per_test": 0.000987, "avg_input_tokens": 292.0, "avg_output_tokens": 1934.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 257.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 21.8, "median_query_seconds": 9.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 31.667, "latency": 57.484, "stderr": 1.782, "cost_per_test": 0.000582, "avg_input_tokens": 131.9, "avg_output_tokens": 22642.2, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 57.5, "median_query_seconds": 44.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 31.25, "latency": 106.797, "stderr": 1.659, "cost_per_test": 0.006307, "avg_input_tokens": 133.6, "avg_output_tokens": 4214.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 106.8, "median_query_seconds": 97.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 30.417, "latency": 76.609, "stderr": 1.327, "cost_per_test": 0.057641, "avg_input_tokens": 136.8, "avg_output_tokens": 7170.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 76.6, "median_query_seconds": 24.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 27.083, "latency": 10.326, "stderr": 0.984, "cost_per_test": 0.000951, "avg_input_tokens": 131.9, "avg_output_tokens": 2344.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 10.3, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 24.583, "latency": 51.322, "stderr": 0.61, "cost_per_test": 0.002953, "avg_input_tokens": 129.3, "avg_output_tokens": 3151.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 51.3, "median_query_seconds": 28.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 24.167, "latency": 10.759, "stderr": 1.045, "cost_per_test": 0.001103, "avg_input_tokens": 136.8, "avg_output_tokens": 2723.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 10.8, "median_query_seconds": 9.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 21.25, "latency": 20.313, "stderr": 3.33, "cost_per_test": 0.003516, "avg_input_tokens": 131.9, "avg_output_tokens": 8757.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 20.3, "median_query_seconds": 17.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 18.333, "latency": 18.429, "stderr": 2.357, "cost_per_test": 0.022458, "avg_input_tokens": 152.6, "avg_output_tokens": 1466.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 18.4, "median_query_seconds": 14.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 17.5, "latency": 5.567, "stderr": 1.22, "cost_per_test": 0.00026, "avg_input_tokens": 131.9, "avg_output_tokens": 834.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 5.6, "median_query_seconds": 5.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 17.5, "latency": 15.575, "stderr": 0.833, "cost_per_test": 0.001154, "avg_input_tokens": 134.2, "avg_output_tokens": 1277.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 15.6, "median_query_seconds": 15.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 15.0, "latency": 35.226, "stderr": 1.26, "cost_per_test": 0.010184, "avg_input_tokens": 136.8, "avg_output_tokens": 984.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 35.2, "median_query_seconds": 18.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 14.167, "latency": 17.102, "stderr": 1.373, "cost_per_test": 0.000678, "avg_input_tokens": 136.8, "avg_output_tokens": 1096.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 17.1, "median_query_seconds": 15.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 13.75, "latency": 10.626, "stderr": 0.755, "cost_per_test": 0.004196, "avg_input_tokens": 131.9, "avg_output_tokens": 806.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 10.6, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 11.667, "latency": 23.196, "stderr": 1.091, "cost_per_test": 0.010479, "avg_input_tokens": 137.1, "avg_output_tokens": 1013.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 23.2, "median_query_seconds": 17.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "grok/grok-2-1212": { "accuracy": 10.833, "latency": 57.739, "stderr": 0.833, "cost_per_test": 0.015694, "avg_input_tokens": 133.2, "avg_output_tokens": 1542.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 57.7, "median_query_seconds": 34.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 10.0, "latency": 21.813, "stderr": 0.891, "cost_per_test": 0.000837, "avg_input_tokens": 134.2, "avg_output_tokens": 1377.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 21.8, "median_query_seconds": 20.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 9.583, "latency": 15.593, "stderr": 1.598, "cost_per_test": 0.011992, "avg_input_tokens": 136.8, "avg_output_tokens": 1165.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 15.6, "median_query_seconds": 15.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 7.917, "latency": 11.042, "stderr": 1.659, "cost_per_test": 0.001123, "avg_input_tokens": 164.8, "avg_output_tokens": 1111.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 11.0, "median_query_seconds": 9.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 6.667, "latency": 17.509, "stderr": 1.992, "cost_per_test": 0.0057, "avg_input_tokens": 142.7, "avg_output_tokens": 902.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 17.5, "median_query_seconds": 14.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 5.417, "latency": 15.609, "stderr": 0.877, "cost_per_test": 0.0, "temperature": 0.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 4.583, "latency": 9.395, "stderr": 1.08, "cost_per_test": 0.008997, "avg_input_tokens": 152.6, "avg_output_tokens": 569.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 9.4, "median_query_seconds": 9.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 3.333, "latency": 8.888, "stderr": 1.091, "cost_per_test": 0.000531, "avg_input_tokens": 133.5, "avg_output_tokens": 840.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 8.9, "median_query_seconds": 7.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 3.333, "latency": 10.225, "stderr": 0.0, "cost_per_test": 0.000246, "avg_input_tokens": 133.5, "avg_output_tokens": 787.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 10.2, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 0.0, "latency": 6.837, "stderr": 0.0, "cost_per_test": 0.000473, "avg_input_tokens": 149.9, "avg_output_tokens": 1108.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 6.9, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 0.0, "latency": 9.352, "stderr": 0.0, "cost_per_test": 0.00252, "avg_input_tokens": 152.6, "avg_output_tokens": 599.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 9.4, "median_query_seconds": 9.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 0.0, "latency": 18.965, "stderr": 0.0, "cost_per_test": 0.007693, "avg_input_tokens": 149.9, "avg_output_tokens": 924.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 30, "avg_turns": 1.0, "avg_query_seconds": 19.0, "median_query_seconds": 18.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } } } }