{ "metadata": { "benchmark": "GPQA Diamond", "slug": "gpqa", "description": "Graduate-level Google-Proof Q&A benchmark evaluating models on questions that require deep reasoning.", "benchmark_id": "gpqa", "family": "gpqa", "version": "1", "updated": "2026-09-01", "dataset_type": "public", "industry": "academic", "tasks": { "overall": "Overall", "diamond_few_shot_cot": "Few-Shot CoT", "diamond_zero_shot_cot": "Zero-Shot CoT" }, "models": [ "ai21labs/jamba-large-1.6", "ai21labs/jamba-mini-1.6", "alibaba/qwen3-max", "alibaba/qwen3-max-2026-01-23", "alibaba/qwen3-max-preview", "alibaba/qwen3.5-flash", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "ant/ling-3.0-flash-2607", "anthropic/claude-3-5-haiku-20241022", "anthropic/claude-3-5-sonnet-20241022", "anthropic/claude-3-7-sonnet-20250219", "anthropic/claude-3-7-sonnet-20250219-thinking", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-1-20250805-thinking", "anthropic/claude-opus-4-20250514", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "cohere/command-a-03-2025", "cohere/command-r-plus", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "fireworks/deepseek-v3", "fireworks/deepseek-v3-0324", "fireworks/deepseek-v3p2", "fireworks/deepseek-v3p2-thinking", "fireworks/gpt-oss-120b", "fireworks/gpt-oss-20b", "fireworks/llama4-maverick-instruct-basic", "fireworks/qwen3-235b-a22b", "google/gemini-1.5-flash-002", "google/gemini-1.5-pro-002", "google/gemini-2.0-flash-001", "google/gemini-2.5-flash-lite-preview-09-2025", "google/gemini-2.5-flash-lite-preview-09-2025-thinking", "google/gemini-2.5-flash-preview-04-17", "google/gemini-2.5-flash-preview-09-2025", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-pro-exp-03-25", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "grok/grok-2-1212", "grok/grok-3", "grok/grok-3-mini-fast-high-reasoning", "grok/grok-3-mini-fast-low-reasoning", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "kimi/kimi-k2-thinking", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse_spark", "meta/muse_spark_1_1", "minimax/MiniMax-M2.1", "minimax/MiniMax-M2.5", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/magistral-medium-2509", "mistralai/magistral-small-2509", "mistralai/mistral-large-2411", "mistralai/mistral-large-2512", "mistralai/mistral-medium-3.5", "mistralai/mistral-small-2402", "mistralai/mistral-small-2503", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-3.5-turbo", "openai/gpt-4.1-2025-04-14", "openai/gpt-4.1-mini-2025-04-14", "openai/gpt-4.1-nano-2025-04-14", "openai/gpt-4o-2024-05-13", "openai/gpt-4o-2024-11-20", "openai/gpt-4o-mini-2024-07-18", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-nano-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/o1-2024-12-17", "openai/o3-2025-04-16", "openai/o3-mini-2025-01-31", "openai/o4-mini-2025-04-16", "poolside/laguna-m.1", "poolside/laguna-xs.2", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561", "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking", "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "together/meta-llama/Llama-4-Scout-17B-16E-Instruct", "together/moonshotai/Kimi-K2-Instruct", "xiaomi/mimo-v2-flash", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "zai/glm-4.5", "zai/glm-4.6", "zai/glm-4.7", "zai/glm-5-thinking", "zai/glm-5.1", "zai/glm-5.2", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "platform", "mode": "one-shot", "archived": false, "partner": false, "total_models": 138 }, "tasks": { "overall": { "google/gemini-3.1-pro-preview": { "accuracy": 95.454, "latency": 65.76, "stderr": 1.046, "cost_per_test": 0.081442, "avg_input_tokens": 1082.6, "avg_output_tokens": 6606.4, "avg_reasoning_tokens": 5931.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 65.8, "median_query_seconds": 41.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 95.202, "latency": 58.304, "stderr": 1.074, "cost_per_test": 0.10996, "avg_input_tokens": 1314.6, "avg_output_tokens": 3526.8, "avg_reasoning_tokens": 3375.6, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 58.3, "median_query_seconds": 20.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 94.697, "latency": 209.056, "stderr": 1.125, "cost_per_test": 0.045365, "avg_input_tokens": 1065.7, "avg_output_tokens": 7319.3, "avg_reasoning_tokens": 7056.5, "avg_cache_read_tokens": 454.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 209.0, "median_query_seconds": 95.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 94.444, "latency": 20.18, "stderr": 1.477, "cost_per_test": 0.056302, "avg_input_tokens": 1082.6, "avg_output_tokens": 7290.4, "avg_reasoning_tokens": 6673.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 20.2, "median_query_seconds": 10.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 93.94, "latency": 8.773, "stderr": 1.488, "cost_per_test": 0.023122, "avg_input_tokens": 1082.6, "avg_output_tokens": 2866.5, "avg_reasoning_tokens": 2344.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 8.8, "median_query_seconds": 6.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 93.686, "latency": 266.606, "stderr": 1.221, "cost_per_test": 0.077028, "avg_input_tokens": 1163.5, "avg_output_tokens": 12595.7, "avg_reasoning_tokens": 12202.7, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 266.6, "median_query_seconds": 137.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 93.434, "latency": 18.064, "stderr": 1.329, "cost_per_test": 0.03189, "avg_input_tokens": 1082.6, "avg_output_tokens": 4035.4, "avg_reasoning_tokens": 3470.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 18.1, "median_query_seconds": 12.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 93.434, "latency": 39.588, "stderr": 1.244, "cost_per_test": 0.095481, "avg_input_tokens": 1571.0, "avg_output_tokens": 3433.6, "avg_reasoning_tokens": 2733.7, "avg_cache_write_tokens": 1430.2, "turn_metrics": { "tasks_measured": 196, "avg_turns": 1.0, "avg_query_seconds": 39.6, "median_query_seconds": 13.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 93.434, "latency": 45.857, "stderr": 1.863, "cost_per_test": 0.204523, "avg_input_tokens": 1579.0, "avg_output_tokens": 3703.0, "avg_cache_write_tokens": 1434.0, "turn_metrics": { "tasks_measured": 197, "avg_turns": 1.0, "avg_query_seconds": 45.8, "median_query_seconds": 16.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 93.182, "latency": 99.902, "stderr": 1.939, "cost_per_test": 0.450473, "avg_input_tokens": 1572.6, "avg_output_tokens": 8733.8, "avg_cache_write_tokens": 1395.8, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 99.9, "median_query_seconds": 30.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 93.182, "latency": 112.326, "stderr": 1.288, "cost_per_test": 0.082767, "avg_input_tokens": 1856.2, "avg_output_tokens": 4379.0, "avg_reasoning_tokens": 4153.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 112.4, "median_query_seconds": 58.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 92.929, "latency": 122.425, "stderr": 1.309, "cost_per_test": 0.088491, "avg_input_tokens": 1145.8, "avg_output_tokens": 5855.8, "avg_reasoning_tokens": 5597.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 122.4, "median_query_seconds": 47.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.5": { "accuracy": 92.929, "latency": 217.858, "stderr": 1.288, "cost_per_test": 0.044029, "avg_input_tokens": 1065.7, "avg_output_tokens": 7165.4, "avg_reasoning_tokens": 6810.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 217.8, "median_query_seconds": 165.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 92.676, "latency": 27.655, "stderr": 1.462, "cost_per_test": 0.063842, "avg_input_tokens": 1081.2, "avg_output_tokens": 5140.0, "avg_reasoning_tokens": 4495.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 27.7, "median_query_seconds": 18.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 92.676, "latency": 278.908, "stderr": 1.443, "cost_per_test": 0.0255, "avg_input_tokens": 1191.7, "avg_output_tokens": 10487.6, "avg_cache_read_tokens": 802.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 278.9, "median_query_seconds": 64.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 92.424, "latency": 133.876, "stderr": 1.984, "cost_per_test": 0.339261, "avg_input_tokens": 1570.4, "avg_output_tokens": 13291.2, "avg_cache_write_tokens": 1396.2, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 133.9, "median_query_seconds": 42.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 92.424, "latency": 279.412, "stderr": 2.02, "cost_per_test": 0.065819, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 91.666, "latency": 53.521, "stderr": 1.736, "cost_per_test": 0.010616, "avg_input_tokens": 1207.4, "avg_output_tokens": 8645.1, "avg_reasoning_tokens": 8446.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 53.6, "median_query_seconds": 21.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 91.666, "latency": 61.34, "stderr": 1.389, "cost_per_test": 0.089573, "avg_input_tokens": 1082.6, "avg_output_tokens": 7284.0, "avg_reasoning_tokens": 6399.8, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 61.4, "median_query_seconds": 53.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 91.666, "latency": 173.214, "stderr": 1.837, "cost_per_test": 0.149037, "avg_input_tokens": 2110.5, "avg_output_tokens": 21456.6, "avg_reasoning_tokens": 11039.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 173.2, "median_query_seconds": 46.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 91.666, "latency": 202.953, "stderr": 1.908, "cost_per_test": 0.20934, "avg_input_tokens": 1065.9, "avg_output_tokens": 7047.6, "avg_reasoning_tokens": 6726.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 203.0, "median_query_seconds": 81.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 91.414, "latency": 739.636, "stderr": 1.406, "cost_per_test": 0.025366, "avg_input_tokens": 1019.4, "avg_output_tokens": 9943.2, "avg_reasoning_tokens": 9568.9, "avg_cache_read_tokens": 730.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 739.6, "median_query_seconds": 430.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 91.162, "latency": 75.198, "stderr": 2.104, "cost_per_test": 0.034304, "avg_input_tokens": 1181.0, "avg_output_tokens": 7898.1, "avg_reasoning_tokens": 7369.2, "avg_cache_read_tokens": 672.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 75.2, "median_query_seconds": 37.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 90.909, "latency": 36.764, "stderr": 1.906, "cost_per_test": 0.033997, "avg_input_tokens": 1065.9, "avg_output_tokens": 2742.8, "avg_reasoning_tokens": 2551.3, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 36.8, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 90.152, "latency": 126.611, "stderr": 1.497, "cost_per_test": 0.06149, "avg_input_tokens": 1125.5, "avg_output_tokens": 7823.6, "avg_reasoning_tokens": 7170.6, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 126.6, "median_query_seconds": 63.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 90.152, "latency": 202.52, "stderr": 1.545, "cost_per_test": 0.365129, "avg_input_tokens": 1572.6, "avg_output_tokens": 15934.8, "avg_cache_write_tokens": 1394.6, "turn_metrics": { "tasks_measured": 189, "avg_turns": 1.0, "avg_query_seconds": 202.6, "median_query_seconds": 92.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 89.899, "latency": 79.956, "stderr": 1.694, "cost_per_test": 0.012875, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark": { "accuracy": 89.646, "latency": 94.58, "stderr": 1.546, "cost_per_test": 0.003074, "avg_input_tokens": 1079.0, "avg_output_tokens": 4853.4, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 94.6, "median_query_seconds": 59.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 89.646, "latency": 319.02, "stderr": 1.186, "cost_per_test": 0.294901, "avg_input_tokens": 1223.5, "avg_output_tokens": 19414.8, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 319.0, "median_query_seconds": 103.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 89.394, "latency": 900.378, "stderr": 1.654, "cost_per_test": 0.125847, "avg_input_tokens": 1112.3, "avg_output_tokens": 31559.8, "avg_reasoning_tokens": 31228.8, "avg_cache_read_tokens": 468.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 900.4, "median_query_seconds": 505.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 89.142, "latency": 544.744, "stderr": 1.997, "cost_per_test": 0.080129, "avg_input_tokens": 1067.8, "avg_output_tokens": 19924.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 544.8, "median_query_seconds": 317.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 88.889, "latency": 63.234, "stderr": 2.224, "cost_per_test": 0.077709, "avg_input_tokens": 1573.8, "avg_output_tokens": 4795.8, "avg_cache_write_tokens": 1400.3, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 63.2, "median_query_seconds": 18.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 88.889, "latency": 116.361, "stderr": 1.579, "cost_per_test": 0.046624, "avg_input_tokens": 1167.5, "avg_output_tokens": 15346.7, "avg_reasoning_tokens": 14968.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 116.4, "median_query_seconds": 65.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 88.636, "latency": 59.421, "stderr": 1.594, "cost_per_test": 0.034461, "avg_input_tokens": 1130.6, "avg_output_tokens": 5581.6, "avg_reasoning_tokens": 5163.0, "avg_cache_read_tokens": 716.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 59.4, "median_query_seconds": 44.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-0709": { "accuracy": 88.132, "latency": 115.516, "stderr": 1.625, "cost_per_test": 0.147971, "avg_input_tokens": 1013.9, "avg_output_tokens": 9662.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 115.6, "median_query_seconds": 87.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 88.132, "latency": 210.7, "stderr": 1.851, "cost_per_test": 0.055878, "avg_input_tokens": 1077.7, "avg_output_tokens": 12417.4, "avg_reasoning_tokens": 12003.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 210.7, "median_query_seconds": 73.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 87.879, "latency": 83.84, "stderr": 1.638, "cost_per_test": 0.044346, "avg_input_tokens": 1082.6, "avg_output_tokens": 14601.6, "avg_reasoning_tokens": 13943.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 83.8, "median_query_seconds": 80.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 87.374, "latency": 103.772, "stderr": 1.669, "cost_per_test": 0.018542, "avg_input_tokens": 1125.5, "avg_output_tokens": 7538.4, "avg_reasoning_tokens": 6675.3, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 103.8, "median_query_seconds": 95.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 87.374, "latency": 122.081, "stderr": 1.668, "cost_per_test": 0.022964, "avg_input_tokens": 1125.5, "avg_output_tokens": 7467.0, "avg_reasoning_tokens": 6838.8, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 122.0, "median_query_seconds": 88.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 87.122, "latency": 308.595, "stderr": 1.784, "cost_per_test": 0.16589, "avg_input_tokens": 1073.9, "avg_output_tokens": 17555.8, "avg_cache_read_tokens": 818.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 308.6, "median_query_seconds": 202.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 86.616, "latency": 85.708, "stderr": 1.711, "cost_per_test": 0.074563, "avg_input_tokens": 1065.9, "avg_output_tokens": 7364.5, "avg_reasoning_tokens": 6842.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 85.7, "median_query_seconds": 44.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 86.616, "latency": 171.727, "stderr": 1.953, "cost_per_test": 0.014051, "avg_input_tokens": 1036.7, "avg_output_tokens": 11398.5, "avg_cache_read_tokens": 45.4, "avg_cache_write_tokens": 991.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 171.7, "median_query_seconds": 79.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 86.364, "latency": 131.004, "stderr": 2.126, "cost_per_test": 0.001626, "avg_input_tokens": 1077.7, "avg_output_tokens": 6186.4, "avg_reasoning_tokens": 5715.2, "avg_cache_read_tokens": 16.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 131.0, "median_query_seconds": 49.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 86.112, "latency": 158.471, "stderr": 1.738, "cost_per_test": null, "avg_input_tokens": 1086.6, "avg_output_tokens": 15037.1, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 158.5, "median_query_seconds": 76.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 85.859, "latency": 138.525, "stderr": 2.476, "cost_per_test": 0.712758, "avg_input_tokens": 1235.5, "avg_output_tokens": 9255.7, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 138.6, "median_query_seconds": 121.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 85.606, "latency": 169.716, "stderr": 1.764, "cost_per_test": 0.072708, "avg_input_tokens": 1066.4, "avg_output_tokens": 7137.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 169.7, "median_query_seconds": 96.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 85.606, "latency": 224.664, "stderr": 1.774, "cost_per_test": 0.05711, "avg_input_tokens": 1077.7, "avg_output_tokens": 12717.2, "avg_reasoning_tokens": 12152.6, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 224.6, "median_query_seconds": 105.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 85.606, "latency": 377.864, "stderr": 2.406, "cost_per_test": 0.408666, "avg_input_tokens": 1215.4, "avg_output_tokens": 26964.2, "avg_cache_write_tokens": 760.7, "turn_metrics": { "tasks_measured": 197, "avg_turns": 1.0, "avg_query_seconds": 377.9, "median_query_seconds": 132.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 85.354, "latency": 21.408, "stderr": 1.789, "cost_per_test": 0.003049, "avg_input_tokens": 1123.4, "avg_output_tokens": 5649.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 21.4, "median_query_seconds": 12.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 84.848, "latency": 43.513, "stderr": 2.197, "cost_per_test": 0.003327, "avg_input_tokens": 1114.1, "avg_output_tokens": 14915.8, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 43.5, "median_query_seconds": 15.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 84.848, "latency": 368.073, "stderr": 1.799, "cost_per_test": 0.0741, "avg_input_tokens": 1108.5, "avg_output_tokens": 12128.4, "avg_reasoning_tokens": 10823.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 368.1, "median_query_seconds": 291.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.1": { "accuracy": 84.518, "latency": 249.704, "stderr": 1.821, "cost_per_test": 0.044897, "avg_input_tokens": 1071.0, "avg_output_tokens": 13879.0, "avg_reasoning_tokens": 13404.8, "avg_cache_read_tokens": 733.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 197, "avg_turns": 1.0, "avg_query_seconds": 249.8, "median_query_seconds": 143.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 84.343, "latency": 68.481, "stderr": 1.85, "cost_per_test": 0.002916, "avg_input_tokens": 1162.4, "avg_output_tokens": 5607.3, "avg_reasoning_tokens": 5255.2, "avg_cache_read_tokens": 799.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 68.4, "median_query_seconds": 39.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 84.091, "latency": 65.777, "stderr": 1.861, "cost_per_test": 0.035552, "avg_input_tokens": 1066.4, "avg_output_tokens": 4177.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 65.8, "median_query_seconds": 31.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 84.091, "latency": 151.843, "stderr": 2.135, "cost_per_test": 0.029962, "avg_input_tokens": 1060.8, "avg_output_tokens": 10087.4, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 151.8, "median_query_seconds": 103.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 83.838, "latency": 13.848, "stderr": 1.847, "cost_per_test": 0.01235, "avg_input_tokens": 1082.6, "avg_output_tokens": 4810.1, "avg_reasoning_tokens": 4285.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 13.8, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 83.586, "latency": 259.758, "stderr": 2.036, "cost_per_test": 0.020022, "avg_input_tokens": 1075.1, "avg_output_tokens": 16582.8, "avg_cache_read_tokens": 833.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 259.8, "median_query_seconds": 200.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5-thinking": { "accuracy": 83.333, "latency": 242.523, "stderr": 1.873, "cost_per_test": 0.034091, "avg_input_tokens": 1062.6, "avg_output_tokens": 10673.4, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 242.5, "median_query_seconds": 173.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 83.08, "latency": 63.11, "stderr": 2.457, "cost_per_test": 0.012743, "avg_input_tokens": 1065.9, "avg_output_tokens": 6296.1, "avg_reasoning_tokens": 6110.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 63.1, "median_query_seconds": 28.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 82.828, "latency": 79.884, "stderr": 1.895, "cost_per_test": 0.003413, "avg_input_tokens": 1125.5, "avg_output_tokens": 8251.5, "avg_reasoning_tokens": 7345.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 79.8, "median_query_seconds": 71.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 82.576, "latency": 251.332, "stderr": 2.468, "cost_per_test": 0.006654, "avg_input_tokens": 1347.2, "avg_output_tokens": 7463.0, "avg_reasoning_tokens": 7036.0, "avg_cache_read_tokens": 584.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 251.3, "median_query_seconds": 121.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 82.07, "latency": 171.962, "stderr": 2.258, "cost_per_test": 0.017869, "avg_input_tokens": 1056.7, "avg_output_tokens": 7285.6, "avg_cache_read_tokens": 36.0, "avg_cache_write_tokens": 1020.7, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 172.0, "median_query_seconds": 89.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 81.633, "latency": 117.251, "stderr": 2.254, "cost_per_test": 0.114262, "avg_input_tokens": 1232.9, "avg_output_tokens": 7370.9, "turn_metrics": { "tasks_measured": 196, "avg_turns": 1.0, "avg_query_seconds": 117.2, "median_query_seconds": 115.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 81.566, "latency": 19.483, "stderr": 1.996, "cost_per_test": 0.002894, "avg_input_tokens": 1090.0, "avg_output_tokens": 5281.1, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 19.4, "median_query_seconds": 12.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 81.566, "latency": 89.393, "stderr": 2.054, "cost_per_test": 0.002538, "avg_input_tokens": 1345.1, "avg_output_tokens": 8675.4, "avg_reasoning_tokens": 8382.2, "avg_cache_read_tokens": 583.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 89.4, "median_query_seconds": 57.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 81.06, "latency": 16.464, "stderr": 1.967, "cost_per_test": 0.002298, "avg_input_tokens": 1082.6, "avg_output_tokens": 5472.4, "avg_reasoning_tokens": 4848.8, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 16.4, "median_query_seconds": 11.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 80.808, "latency": 41.102, "stderr": 1.998, "cost_per_test": 0.012944, "avg_input_tokens": 1090.5, "avg_output_tokens": 1158.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 41.1, "median_query_seconds": 29.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 80.303, "latency": 67.133, "stderr": 1.996, "cost_per_test": 0.013593, "avg_input_tokens": 1116.1, "avg_output_tokens": 6657.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 67.1, "median_query_seconds": 66.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 80.303, "latency": 260.248, "stderr": 1.997, "cost_per_test": 0.003465, "avg_input_tokens": 1032.4, "avg_output_tokens": 8372.8, "avg_reasoning_tokens": 8170.4, "avg_cache_read_tokens": 999.8, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 260.2, "median_query_seconds": 193.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-4.7": { "accuracy": 80.05, "latency": 211.079, "stderr": 2.006, "cost_per_test": 0.024866, "avg_input_tokens": 1070.8, "avg_output_tokens": 11172.0, "avg_cache_read_tokens": 724.6, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 211.0, "median_query_seconds": 153.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 79.546, "latency": 13.54, "stderr": 2.463, "cost_per_test": 0.074932, "avg_input_tokens": 1206.5, "avg_output_tokens": 757.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 13.6, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "alibaba/qwen3-max": { "accuracy": 79.546, "latency": 70.152, "stderr": 2.026, "cost_per_test": 0.016466, "avg_input_tokens": 1106.1, "avg_output_tokens": 2523.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 70.2, "median_query_seconds": 45.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 79.293, "latency": 40.618, "stderr": 2.044, "cost_per_test": 0.00388, "avg_input_tokens": 1013.9, "avg_output_tokens": 7152.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 40.6, "median_query_seconds": 34.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 78.536, "latency": 56.176, "stderr": 2.253, "cost_per_test": 0.006212, "avg_input_tokens": 1134.4, "avg_output_tokens": 10070.4, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 56.2, "median_query_seconds": 32.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 78.536, "latency": 135.412, "stderr": 2.128, "cost_per_test": 0.011422, "avg_input_tokens": 1053.7, "avg_output_tokens": 9198.6, "avg_cache_read_tokens": 31.0, "avg_cache_write_tokens": 1020.7, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 135.4, "median_query_seconds": 75.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 78.536, "latency": 585.555, "stderr": 2.183, "cost_per_test": 0.023072, "avg_input_tokens": 1059.6, "avg_output_tokens": 9464.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 585.6, "median_query_seconds": 402.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 77.778, "latency": 106.784, "stderr": 2.09, "cost_per_test": 0.01338, "avg_input_tokens": 1106.1, "avg_output_tokens": 2008.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 106.8, "median_query_seconds": 49.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 77.526, "latency": 29.498, "stderr": 2.417, "cost_per_test": 0.001565, "avg_input_tokens": 1066.2, "avg_output_tokens": 3779.0, "avg_reasoning_tokens": 3408.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 29.5, "median_query_seconds": 13.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 76.516, "latency": 23.53, "stderr": 2.154, "cost_per_test": 0.002833, "avg_input_tokens": 1090.0, "avg_output_tokens": 5201.4, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 23.5, "median_query_seconds": 16.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 76.263, "latency": 35.772, "stderr": 2.16, "cost_per_test": 0.144678, "avg_input_tokens": 1235.5, "avg_output_tokens": 1681.9, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 35.8, "median_query_seconds": 32.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 76.263, "latency": 46.828, "stderr": 2.161, "cost_per_test": 0.00065, "avg_input_tokens": 1033.3, "avg_output_tokens": 1459.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1000.9, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 46.8, "median_query_seconds": 39.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 75.505, "latency": 99.984, "stderr": 2.176, "cost_per_test": 0.035857, "avg_input_tokens": 1066.4, "avg_output_tokens": 7882.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 100.0, "median_query_seconds": 38.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 75.253, "latency": 155.02, "stderr": 2.169, "cost_per_test": 0.18875, "avg_input_tokens": 1235.5, "avg_output_tokens": 12336.3, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 155.0, "median_query_seconds": 149.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 75.0, "latency": 118.458, "stderr": 2.191, "cost_per_test": 0.116706, "avg_input_tokens": 1235.5, "avg_output_tokens": 7533.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 118.4, "median_query_seconds": 125.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 74.495, "latency": 30.004, "stderr": 2.19, "cost_per_test": 0.012961, "avg_input_tokens": 1066.4, "avg_output_tokens": 2679.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 30.0, "median_query_seconds": 17.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.6": { "accuracy": 74.495, "latency": 474.834, "stderr": 2.197, "cost_per_test": 0.01541, "avg_input_tokens": 1072.4, "avg_output_tokens": 6711.9, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 474.8, "median_query_seconds": 89.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-3": { "accuracy": 74.242, "latency": 29.912, "stderr": 2.209, "cost_per_test": 0.039242, "avg_input_tokens": 1012.9, "avg_output_tokens": 2413.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 29.9, "median_query_seconds": 27.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 73.232, "latency": 40.176, "stderr": 2.231, "cost_per_test": 0.252632, "avg_input_tokens": 1066.4, "avg_output_tokens": 3944.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 40.2, "median_query_seconds": 28.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 72.98, "latency": 14.234, "stderr": 2.233, "cost_per_test": 0.001599, "avg_input_tokens": 1013.9, "avg_output_tokens": 2589.4, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 14.2, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 72.222, "latency": 83.203, "stderr": 2.474, "cost_per_test": 0.055064, "avg_input_tokens": 1234.5, "avg_output_tokens": 10765.8, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 83.2, "median_query_seconds": 66.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.5": { "accuracy": 72.222, "latency": 704.224, "stderr": 2.262, "cost_per_test": 0.019454, "avg_input_tokens": 1070.8, "avg_output_tokens": 8550.6, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 704.2, "median_query_seconds": 217.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 71.717, "latency": 19.13, "stderr": 2.261, "cost_per_test": 0.066808, "avg_input_tokens": 1207.5, "avg_output_tokens": 649.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 19.2, "median_query_seconds": 18.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 71.464, "latency": 41.434, "stderr": 2.266, "cost_per_test": 0.004481, "avg_input_tokens": 1077.2, "avg_output_tokens": 1134.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 41.4, "median_query_seconds": 32.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 70.202, "latency": 20.61, "stderr": 2.298, "cost_per_test": 0.000481, "avg_input_tokens": 1090.0, "avg_output_tokens": 6569.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 20.6, "median_query_seconds": 13.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 70.202, "latency": 306.218, "stderr": 2.373, "cost_per_test": 0.006472, "avg_input_tokens": 1106.1, "avg_output_tokens": 7078.8, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 306.2, "median_query_seconds": 171.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 69.95, "latency": 35.934, "stderr": 2.303, "cost_per_test": 0.067421, "avg_input_tokens": 1207.5, "avg_output_tokens": 657.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 35.9, "median_query_seconds": 35.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 69.444, "latency": 10.071, "stderr": 2.35, "cost_per_test": 0.000964, "avg_input_tokens": 1027.0, "avg_output_tokens": 838.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 10.0, "median_query_seconds": 9.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 69.444, "latency": 12.818, "stderr": 2.315, "cost_per_test": 0.013552, "avg_input_tokens": 1207.5, "avg_output_tokens": 662.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 12.8, "median_query_seconds": 11.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 68.94, "latency": 145.092, "stderr": 2.465, "cost_per_test": 0.006738, "avg_input_tokens": 1134.4, "avg_output_tokens": 22195.6, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 145.1, "median_query_seconds": 84.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 68.434, "latency": 21.216, "stderr": 2.372, "cost_per_test": 0.00215, "avg_input_tokens": 1073.6, "avg_output_tokens": 1075.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 21.2, "median_query_seconds": 18.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 67.929, "latency": 9.282, "stderr": 2.351, "cost_per_test": 0.002324, "avg_input_tokens": 1067.4, "avg_output_tokens": 1186.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 9.3, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 67.424, "latency": 9.218, "stderr": 2.354, "cost_per_test": 0.011972, "avg_input_tokens": 1207.5, "avg_output_tokens": 556.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 9.2, "median_query_seconds": 8.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 65.404, "latency": 23.14, "stderr": 2.402, "cost_per_test": 0.013852, "avg_input_tokens": 1067.4, "avg_output_tokens": 1464.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 23.1, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 65.152, "latency": 5.803, "stderr": 2.394, "cost_per_test": 0.00049, "avg_input_tokens": 1090.5, "avg_output_tokens": 953.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 5.8, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 65.152, "latency": 8.66, "stderr": 2.398, "cost_per_test": 0.000445, "avg_input_tokens": 1174.4, "avg_output_tokens": 657.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 792.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 8.7, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 64.142, "latency": 7.274, "stderr": 2.412, "cost_per_test": 0.001236, "avg_input_tokens": 1090.0, "avg_output_tokens": 2816.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 7.3, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 63.384, "latency": 49.716, "stderr": 2.455, "cost_per_test": 0.004578, "avg_input_tokens": 1146.2, "avg_output_tokens": 11301.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 49.8, "median_query_seconds": 41.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 62.374, "latency": 77.589, "stderr": 2.451, "cost_per_test": null, "avg_input_tokens": 1096.5, "avg_output_tokens": 4796.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 77.6, "median_query_seconds": 65.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 62.374, "latency": 94.145, "stderr": 2.435, "cost_per_test": 0.03601, "avg_input_tokens": 1074.0, "avg_output_tokens": 6806.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 94.1, "median_query_seconds": 64.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 62.122, "latency": 3.668, "stderr": 2.429, "cost_per_test": 0.001098, "avg_input_tokens": 1133.4, "avg_output_tokens": 1743.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 3.6, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 61.616, "latency": 23.475, "stderr": 2.447, "cost_per_test": 0.001552, "avg_input_tokens": 1032.8, "avg_output_tokens": 692.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 23.4, "median_query_seconds": 20.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 59.344, "latency": 7.241, "stderr": 2.468, "cost_per_test": 0.009078, "avg_input_tokens": 1207.5, "avg_output_tokens": 363.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 7.2, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2-flash": { "accuracy": 59.344, "latency": 20.902, "stderr": 2.496, "cost_per_test": 0.001212, "avg_input_tokens": 1124.1, "avg_output_tokens": 3901.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 788.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 20.9, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 58.334, "latency": 7.709, "stderr": 2.475, "cost_per_test": 0.003655, "avg_input_tokens": 1090.5, "avg_output_tokens": 458.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 7.7, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 58.334, "latency": 46.252, "stderr": 2.487, "cost_per_test": 0.007058, "avg_input_tokens": 1073.9, "avg_output_tokens": 4359.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 46.2, "median_query_seconds": 39.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 57.576, "latency": 43.452, "stderr": 2.411, "cost_per_test": 0.030117, "avg_input_tokens": 1082.1, "avg_output_tokens": 11916.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 43.4, "median_query_seconds": 8.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 55.05, "latency": 69.312, "stderr": 2.499, "cost_per_test": 0.001373, "avg_input_tokens": 1161.2, "avg_output_tokens": 9351.4, "avg_reasoning_tokens": 6052.2, "avg_cache_read_tokens": 882.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 69.3, "median_query_seconds": 57.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 54.546, "latency": 21.282, "stderr": 2.504, "cost_per_test": 0.001468, "avg_input_tokens": 1032.8, "avg_output_tokens": 598.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 21.2, "median_query_seconds": 18.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 27.02, "latency": 327.957, "stderr": 1.922, "cost_per_test": null, "avg_input_tokens": 1124.2, "avg_output_tokens": 16537.0, "avg_reasoning_tokens": 10456.2, "avg_cache_read_tokens": 874.3, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 267.0, "median_query_seconds": 255.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 53.788, "latency": 14.924, "stderr": 2.507, "cost_per_test": 0.009465, "avg_input_tokens": 1079.4, "avg_output_tokens": 676.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 14.9, "median_query_seconds": 13.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 50.758, "latency": 4.192, "stderr": 2.457, "cost_per_test": 0.000488, "avg_input_tokens": 1067.4, "avg_output_tokens": 952.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 4.2, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 50.758, "latency": 7.986, "stderr": 2.495, "cost_per_test": 0.000586, "avg_input_tokens": 1074.3, "avg_output_tokens": 618.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 8.0, "median_query_seconds": 6.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-2-1212": { "accuracy": 50.758, "latency": 20.178, "stderr": 2.51, "cost_per_test": 0.0095, "avg_input_tokens": 1066.9, "avg_output_tokens": 736.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 20.2, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4o-2024-05-13": { "accuracy": 50.252, "latency": 10.078, "stderr": 2.512, "cost_per_test": 0.01544, "avg_input_tokens": 1067.4, "avg_output_tokens": 673.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 10.1, "median_query_seconds": 8.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 50.0, "latency": 7.342, "stderr": 2.512, "cost_per_test": 0.001608, "avg_input_tokens": 1103.5, "avg_output_tokens": 724.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 7.4, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 48.485, "latency": 11.49, "stderr": 2.276, "cost_per_test": 0.010094, "avg_input_tokens": 1117.3, "avg_output_tokens": 730.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 11.5, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 47.728, "latency": 15.802, "stderr": 2.5, "cost_per_test": 0.006714, "avg_input_tokens": 1256.1, "avg_output_tokens": 700.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 15.8, "median_query_seconds": 11.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 47.727, "latency": 16.482, "stderr": 2.468, "cost_per_test": null, "avg_input_tokens": 1099.9, "avg_output_tokens": 1074.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 16.5, "median_query_seconds": 15.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 46.97, "latency": 12.092, "stderr": 2.491, "cost_per_test": 0.000654, "avg_input_tokens": 1027.0, "avg_output_tokens": 796.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 12.1, "median_query_seconds": 11.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 45.96, "latency": 3.39, "stderr": 2.504, "cost_per_test": 0.000233, "avg_input_tokens": 1090.5, "avg_output_tokens": 502.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 3.4, "median_query_seconds": 3.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 44.192, "latency": 8.824, "stderr": 2.473, "cost_per_test": 0.000278, "avg_input_tokens": 1074.1, "avg_output_tokens": 660.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 8.8, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 44.192, "latency": 15.32, "stderr": 2.495, "cost_per_test": 0.000536, "avg_input_tokens": 1067.4, "avg_output_tokens": 627.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 15.3, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 37.879, "latency": 6.56, "stderr": 2.436, "cost_per_test": 0.00222, "avg_input_tokens": 1207.5, "avg_output_tokens": 313.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 6.6, "median_query_seconds": 6.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 36.111, "latency": 15.471, "stderr": 2.099, "cost_per_test": 0.00669, "avg_input_tokens": 1164.8, "avg_output_tokens": 545.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 15.5, "median_query_seconds": 14.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 34.848, "latency": 256.17, "stderr": 2.39, "cost_per_test": 0.095489, "avg_input_tokens": 1071.0, "avg_output_tokens": 12517.6, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 256.2, "median_query_seconds": 239.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 32.07, "latency": 4.953, "stderr": 1.988, "cost_per_test": 0.000487, "avg_input_tokens": 1164.8, "avg_output_tokens": 635.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 5.0, "median_query_seconds": 4.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "cohere/command-r-plus": { "accuracy": 31.06, "latency": 8.194, "stderr": 2.28, "cost_per_test": 0.007438, "avg_input_tokens": 1117.3, "avg_output_tokens": 464.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 8.2, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "openai/gpt-3.5-turbo": { "accuracy": 30.556, "latency": 2.628, "stderr": 2.287, "cost_per_test": 0.000862, "avg_input_tokens": 1076.6, "avg_output_tokens": 215.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 2.6, "median_query_seconds": 2.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null } }, "diamond_few_shot_cot": { "google/gemini-3.1-pro-preview": { "accuracy": 95.96, "latency": 67.495, "stderr": 1.399, "cost_per_test": 0.085026, "avg_input_tokens": 1909.1, "avg_output_tokens": 6767.3, "avg_reasoning_tokens": 6121.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 67.5, "median_query_seconds": 42.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 95.455, "latency": 9.186, "stderr": 1.76, "cost_per_test": 0.025355, "avg_input_tokens": 1909.1, "avg_output_tokens": 2998.9, "avg_reasoning_tokens": 2471.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 9.2, "median_query_seconds": 6.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 94.949, "latency": 21.842, "stderr": 1.881, "cost_per_test": 0.061477, "avg_input_tokens": 1909.1, "avg_output_tokens": 7815.2, "avg_reasoning_tokens": 7186.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 21.8, "median_query_seconds": 11.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 94.949, "latency": 72.403, "stderr": 1.556, "cost_per_test": 0.135633, "avg_input_tokens": 1991.4, "avg_output_tokens": 4350.2, "avg_reasoning_tokens": 4085.1, "avg_cache_read_tokens": 1073.1, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 72.4, "median_query_seconds": 31.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 93.939, "latency": 211.173, "stderr": 1.696, "cost_per_test": 0.046346, "avg_input_tokens": 1852.7, "avg_output_tokens": 7333.9, "avg_reasoning_tokens": 7066.4, "avg_cache_read_tokens": 908.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 211.2, "median_query_seconds": 97.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 93.434, "latency": 118.4, "stderr": 1.822, "cost_per_test": 0.092359, "avg_input_tokens": 2622.8, "avg_output_tokens": 4874.3, "avg_reasoning_tokens": 4504.1, "avg_cache_read_tokens": 1202.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 118.4, "median_query_seconds": 66.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 93.434, "latency": 129.475, "stderr": 1.76, "cost_per_test": 0.093316, "avg_input_tokens": 1952.3, "avg_output_tokens": 6146.9, "avg_reasoning_tokens": 5832.4, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 129.5, "median_query_seconds": 47.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 92.929, "latency": 47.631, "stderr": 1.822, "cost_per_test": 0.22434, "avg_input_tokens": 2773.8, "avg_output_tokens": 3793.6, "avg_cache_write_tokens": 2769.5, "turn_metrics": { "tasks_measured": 197, "avg_turns": 1.0, "avg_query_seconds": 47.6, "median_query_seconds": 16.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 92.929, "latency": 132.309, "stderr": 2.043, "cost_per_test": 0.610919, "avg_input_tokens": 2768.8, "avg_output_tokens": 11714.3, "avg_cache_write_tokens": 2766.8, "turn_metrics": { "tasks_measured": 195, "avg_turns": 1.0, "avg_query_seconds": 132.3, "median_query_seconds": 39.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 92.929, "latency": 288.128, "stderr": 1.822, "cost_per_test": 0.046385, "avg_input_tokens": 1852.7, "avg_output_tokens": 7478.2, "avg_reasoning_tokens": 7118.9, "avg_cache_read_tokens": 1459.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 288.1, "median_query_seconds": 226.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.3": { "accuracy": 92.929, "latency": 396.598, "stderr": 1.822, "cost_per_test": 0.024086, "avg_input_tokens": 1787.9, "avg_output_tokens": 9353.2, "avg_reasoning_tokens": 8973.1, "avg_cache_read_tokens": 1458.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 396.6, "median_query_seconds": 219.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 92.424, "latency": 16.872, "stderr": 1.991, "cost_per_test": 0.031197, "avg_input_tokens": 1909.1, "avg_output_tokens": 3777.8, "avg_reasoning_tokens": 3242.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 16.9, "median_query_seconds": 12.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 92.424, "latency": 38.585, "stderr": 1.881, "cost_per_test": 0.100176, "avg_input_tokens": 2768.5, "avg_output_tokens": 3315.0, "avg_reasoning_tokens": 2606.2, "avg_cache_write_tokens": 2766.5, "turn_metrics": { "tasks_measured": 196, "avg_turns": 1.0, "avg_query_seconds": 38.6, "median_query_seconds": 14.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 92.424, "latency": 45.557, "stderr": 2.319, "cost_per_test": 0.009378, "avg_input_tokens": 1872.4, "avg_output_tokens": 7503.3, "avg_reasoning_tokens": 7255.7, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 45.6, "median_query_seconds": 16.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 92.424, "latency": 244.029, "stderr": 1.881, "cost_per_test": 0.023, "avg_input_tokens": 1967.7, "avg_output_tokens": 9383.2, "avg_cache_read_tokens": 1459.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 244.0, "median_query_seconds": 62.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 92.424, "latency": 265.896, "stderr": 1.881, "cost_per_test": 0.077068, "avg_input_tokens": 2016.0, "avg_output_tokens": 12463.9, "avg_reasoning_tokens": 11968.3, "avg_cache_read_tokens": 998.1, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 265.9, "median_query_seconds": 125.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 91.919, "latency": 131.379, "stderr": 2.277, "cost_per_test": 0.3381, "avg_input_tokens": 2769.3, "avg_output_tokens": 12971.2, "avg_cache_write_tokens": 2767.3, "turn_metrics": { "tasks_measured": 194, "avg_turns": 1.0, "avg_query_seconds": 131.4, "median_query_seconds": 39.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 91.919, "latency": 170.268, "stderr": 2.513, "cost_per_test": 0.149071, "avg_input_tokens": 2797.9, "avg_output_tokens": 21168.3, "avg_reasoning_tokens": 10799.0, "avg_cache_read_tokens": 631.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 197, "avg_turns": 1.0, "avg_query_seconds": 170.3, "median_query_seconds": 51.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 91.919, "latency": 278.283, "stderr": 3.024, "cost_per_test": 0.061962, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 91.414, "latency": 26.753, "stderr": 1.991, "cost_per_test": 0.066225, "avg_input_tokens": 1906.3, "avg_output_tokens": 5201.1, "avg_reasoning_tokens": 4560.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 197, "avg_turns": 1.0, "avg_query_seconds": 26.8, "median_query_seconds": 18.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 91.414, "latency": 186.909, "stderr": 3.127, "cost_per_test": 0.196199, "avg_input_tokens": 1872.4, "avg_output_tokens": 6587.0, "avg_reasoning_tokens": 6172.9, "avg_cache_read_tokens": 476.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 186.9, "median_query_seconds": 82.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 90.909, "latency": 58.765, "stderr": 2.043, "cost_per_test": 0.087749, "avg_input_tokens": 1909.1, "avg_output_tokens": 6994.3, "avg_reasoning_tokens": 6121.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 58.8, "median_query_seconds": 50.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 90.909, "latency": 203.874, "stderr": 2.043, "cost_per_test": 0.352728, "avg_input_tokens": 2769.2, "avg_output_tokens": 15504.4, "avg_cache_write_tokens": 2763.2, "turn_metrics": { "tasks_measured": 189, "avg_turns": 1.0, "avg_query_seconds": 203.9, "median_query_seconds": 99.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 90.404, "latency": 67.934, "stderr": 2.616, "cost_per_test": 0.030784, "avg_input_tokens": 1957.5, "avg_output_tokens": 6978.1, "avg_reasoning_tokens": 6380.3, "avg_cache_read_tokens": 1200.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 67.9, "median_query_seconds": 35.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 90.404, "latency": 823.036, "stderr": 2.277, "cost_per_test": 0.112916, "avg_input_tokens": 1896.3, "avg_output_tokens": 28167.1, "avg_reasoning_tokens": 27794.6, "avg_cache_read_tokens": 884.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 823.0, "median_query_seconds": 416.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 89.899, "latency": 43.438, "stderr": 2.711, "cost_per_test": 0.033269, "avg_input_tokens": 1872.4, "avg_output_tokens": 2634.9, "avg_reasoning_tokens": 2349.1, "avg_cache_read_tokens": 1163.6, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 43.4, "median_query_seconds": 12.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 89.899, "latency": 80.111, "stderr": 2.233, "cost_per_test": 0.012966, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 89.899, "latency": 82.709, "stderr": 2.142, "cost_per_test": 0.044792, "avg_input_tokens": 1978.0, "avg_output_tokens": 5312.9, "avg_reasoning_tokens": 4600.9, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 82.7, "median_query_seconds": 63.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 89.899, "latency": 500.4, "stderr": 2.277, "cost_per_test": 0.075267, "avg_input_tokens": 1874.3, "avg_output_tokens": 18662.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1473.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 500.4, "median_query_seconds": 285.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 88.889, "latency": 103.08, "stderr": 2.233, "cost_per_test": 0.044773, "avg_input_tokens": 2020.0, "avg_output_tokens": 14587.6, "avg_reasoning_tokens": 14127.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 103.1, "median_query_seconds": 57.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 88.889, "latency": 205.365, "stderr": 2.36, "cost_per_test": 0.058921, "avg_input_tokens": 1888.2, "avg_output_tokens": 12912.0, "avg_reasoning_tokens": 12414.3, "avg_cache_read_tokens": 469.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 205.4, "median_query_seconds": 70.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark": { "accuracy": 88.384, "latency": 94.376, "stderr": 2.277, "cost_per_test": 0.003188, "avg_input_tokens": 1855.5, "avg_output_tokens": 4850.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 94.4, "median_query_seconds": 56.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 88.384, "latency": 98.941, "stderr": 2.277, "cost_per_test": 0.017758, "avg_input_tokens": 1978.0, "avg_output_tokens": 5589.5, "avg_reasoning_tokens": 5016.4, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 98.9, "median_query_seconds": 89.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 88.384, "latency": 337.27, "stderr": 1.115, "cost_per_test": 0.310954, "avg_input_tokens": 2135.5, "avg_output_tokens": 20301.9, "avg_cache_write_tokens": 25.3, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 337.3, "median_query_seconds": 107.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 87.879, "latency": 119.018, "stderr": 2.319, "cost_per_test": 0.017962, "avg_input_tokens": 1978.0, "avg_output_tokens": 7154.6, "avg_reasoning_tokens": 6304.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 119.0, "median_query_seconds": 109.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-0709": { "accuracy": 87.879, "latency": 119.477, "stderr": 2.319, "cost_per_test": 0.152299, "avg_input_tokens": 1782.9, "avg_output_tokens": 9796.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 119.5, "median_query_seconds": 92.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 87.879, "latency": 281.977, "stderr": 2.319, "cost_per_test": 0.152118, "avg_input_tokens": 1880.4, "avg_output_tokens": 15997.2, "avg_cache_read_tokens": 1553.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 282.0, "median_query_seconds": 123.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 87.374, "latency": 55.38, "stderr": 3.456, "cost_per_test": 0.084904, "avg_input_tokens": 2770.6, "avg_output_tokens": 4967.7, "avg_cache_write_tokens": 2768.6, "turn_metrics": { "tasks_measured": 195, "avg_turns": 1.0, "avg_query_seconds": 55.4, "median_query_seconds": 15.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 87.374, "latency": 60.125, "stderr": 2.36, "cost_per_test": 0.035734, "avg_input_tokens": 1898.9, "avg_output_tokens": 5717.7, "avg_reasoning_tokens": 5303.3, "avg_cache_read_tokens": 1316.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 60.1, "median_query_seconds": 44.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 86.364, "latency": 83.303, "stderr": 2.439, "cost_per_test": 0.077042, "avg_input_tokens": 1872.4, "avg_output_tokens": 7553.0, "avg_reasoning_tokens": 6795.4, "avg_cache_read_tokens": 737.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 83.3, "median_query_seconds": 48.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 86.364, "latency": 85.167, "stderr": 2.439, "cost_per_test": 0.04534, "avg_input_tokens": 1909.1, "avg_output_tokens": 14795.1, "avg_reasoning_tokens": 14137.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 85.2, "median_query_seconds": 80.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 86.364, "latency": 139.968, "stderr": 3.501, "cost_per_test": 0.733867, "avg_input_tokens": 2147.5, "avg_output_tokens": 9354.1, "avg_cache_write_tokens": 25.3, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 140.0, "median_query_seconds": 126.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 86.364, "latency": 142.062, "stderr": 2.439, "cost_per_test": null, "avg_input_tokens": 1904.1, "avg_output_tokens": 14023.3, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 142.1, "median_query_seconds": 68.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 85.859, "latency": 364.556, "stderr": 3.397, "cost_per_test": 0.398667, "avg_input_tokens": 2120.9, "avg_output_tokens": 26080.3, "avg_cache_read_tokens": 11.0, "avg_cache_write_tokens": 1505.2, "turn_metrics": { "tasks_measured": 197, "avg_turns": 1.0, "avg_query_seconds": 364.6, "median_query_seconds": 129.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 85.787, "latency": 197.756, "stderr": 2.488, "cost_per_test": 0.034962, "avg_input_tokens": 1881.7, "avg_output_tokens": 10703.9, "avg_reasoning_tokens": 10064.2, "avg_cache_read_tokens": 1465.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 197, "avg_turns": 1.0, "avg_query_seconds": 197.8, "median_query_seconds": 97.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 85.354, "latency": 38.322, "stderr": 3.296, "cost_per_test": 0.002799, "avg_input_tokens": 1950.6, "avg_output_tokens": 12406.8, "avg_cache_read_tokens": 1277.4, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 38.3, "median_query_seconds": 13.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 85.354, "latency": 117.31, "stderr": 2.799, "cost_per_test": 0.001594, "avg_input_tokens": 1888.2, "avg_output_tokens": 5815.5, "avg_reasoning_tokens": 5256.4, "avg_cache_read_tokens": 28.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 117.3, "median_query_seconds": 47.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 85.354, "latency": 145.346, "stderr": 2.93, "cost_per_test": 0.01282, "avg_input_tokens": 1812.7, "avg_output_tokens": 10121.8, "avg_cache_read_tokens": 16.0, "avg_cache_write_tokens": 1796.7, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 145.3, "median_query_seconds": 54.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 84.848, "latency": 60.078, "stderr": 3.55, "cost_per_test": 0.012277, "avg_input_tokens": 1872.4, "avg_output_tokens": 6020.1, "avg_reasoning_tokens": 5723.9, "avg_cache_read_tokens": 1029.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 60.1, "median_query_seconds": 25.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 84.848, "latency": 191.702, "stderr": 2.548, "cost_per_test": 0.074134, "avg_input_tokens": 1873.4, "avg_output_tokens": 7179.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 191.7, "median_query_seconds": 119.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 84.343, "latency": 22.395, "stderr": 2.583, "cost_per_test": 0.003746, "avg_input_tokens": 1891.9, "avg_output_tokens": 6735.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 22.4, "median_query_seconds": 12.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 84.343, "latency": 163.291, "stderr": 3.375, "cost_per_test": 0.017314, "avg_input_tokens": 1832.7, "avg_output_tokens": 6933.0, "avg_cache_read_tokens": 36.0, "avg_cache_write_tokens": 1796.7, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 163.3, "median_query_seconds": 85.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 83.838, "latency": 70.139, "stderr": 2.616, "cost_per_test": 0.003089, "avg_input_tokens": 1930.9, "avg_output_tokens": 5837.1, "avg_reasoning_tokens": 5467.5, "avg_cache_read_tokens": 1436.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 70.1, "median_query_seconds": 38.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.2": { "accuracy": 83.333, "latency": 178.184, "stderr": 2.649, "cost_per_test": 0.049779, "avg_input_tokens": 1888.2, "avg_output_tokens": 10873.8, "avg_reasoning_tokens": 10280.0, "avg_cache_read_tokens": 622.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 178.2, "median_query_seconds": 94.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 82.828, "latency": 74.323, "stderr": 2.68, "cost_per_test": 0.037891, "avg_input_tokens": 1873.4, "avg_output_tokens": 4268.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 74.3, "median_query_seconds": 33.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 82.828, "latency": 122.006, "stderr": 2.68, "cost_per_test": 0.024807, "avg_input_tokens": 1861.6, "avg_output_tokens": 8217.4, "avg_cache_read_tokens": 1418.4, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 122.0, "median_query_seconds": 88.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 82.828, "latency": 234.216, "stderr": 2.68, "cost_per_test": 0.031964, "avg_input_tokens": 1880.9, "avg_output_tokens": 9930.7, "avg_cache_read_tokens": 1294.8, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 234.2, "median_query_seconds": 165.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 82.828, "latency": 351.668, "stderr": 2.68, "cost_per_test": 0.07229, "avg_input_tokens": 1946.6, "avg_output_tokens": 11659.0, "avg_reasoning_tokens": 10341.1, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 351.7, "median_query_seconds": 274.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 82.323, "latency": 19.829, "stderr": 2.711, "cost_per_test": 0.002954, "avg_input_tokens": 1923.5, "avg_output_tokens": 5375.0, "avg_reasoning_tokens": 4424.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 19.8, "median_query_seconds": 11.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 82.323, "latency": 77.318, "stderr": 2.711, "cost_per_test": 0.003293, "avg_input_tokens": 1978.0, "avg_output_tokens": 7738.6, "avg_reasoning_tokens": 6862.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 77.3, "median_query_seconds": 69.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 81.818, "latency": 13.94, "stderr": 2.741, "cost_per_test": 0.012617, "avg_input_tokens": 1909.1, "avg_output_tokens": 4817.7, "avg_reasoning_tokens": 4286.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 13.9, "median_query_seconds": 10.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 81.633, "latency": 108.446, "stderr": 3.363, "cost_per_test": 0.108671, "avg_input_tokens": 2144.9, "avg_output_tokens": 6815.8, "turn_metrics": { "tasks_measured": 196, "avg_turns": 1.0, "avg_query_seconds": 108.4, "median_query_seconds": 102.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 80.808, "latency": 237.64, "stderr": 3.429, "cost_per_test": 0.006645, "avg_input_tokens": 2186.4, "avg_output_tokens": 7186.4, "avg_reasoning_tokens": 6750.4, "avg_cache_read_tokens": 977.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 237.6, "median_query_seconds": 133.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 80.303, "latency": 39.776, "stderr": 2.853, "cost_per_test": 0.014206, "avg_input_tokens": 1924.5, "avg_output_tokens": 1180.1, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 39.8, "median_query_seconds": 29.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 80.303, "latency": 40.258, "stderr": 2.826, "cost_per_test": 0.004123, "avg_input_tokens": 1782.9, "avg_output_tokens": 7176.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 40.3, "median_query_seconds": 34.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 79.798, "latency": 77.418, "stderr": 2.905, "cost_per_test": 0.002294, "avg_input_tokens": 2183.6, "avg_output_tokens": 7576.7, "avg_reasoning_tokens": 7193.7, "avg_cache_read_tokens": 974.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 77.4, "median_query_seconds": 53.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 79.798, "latency": 241.348, "stderr": 2.905, "cost_per_test": 0.018587, "avg_input_tokens": 1882.8, "avg_output_tokens": 15331.8, "avg_cache_read_tokens": 1567.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 179, "avg_turns": 1.0, "avg_query_seconds": 241.3, "median_query_seconds": 174.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 79.293, "latency": 14.455, "stderr": 3.494, "cost_per_test": 0.092902, "avg_input_tokens": 2118.5, "avg_output_tokens": 813.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 25.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 14.5, "median_query_seconds": 14.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 79.293, "latency": 16.085, "stderr": 2.88, "cost_per_test": 0.002329, "avg_input_tokens": 1909.1, "avg_output_tokens": 5344.0, "avg_reasoning_tokens": 4744.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 16.1, "median_query_seconds": 10.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max": { "accuracy": 78.788, "latency": 68.635, "stderr": 2.905, "cost_per_test": 0.017168, "avg_input_tokens": 1944.6, "avg_output_tokens": 2472.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 68.6, "median_query_seconds": 41.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 78.788, "latency": 246.265, "stderr": 2.905, "cost_per_test": 0.003266, "avg_input_tokens": 1816.0, "avg_output_tokens": 7878.8, "avg_reasoning_tokens": 7653.4, "avg_cache_read_tokens": 1783.7, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 246.3, "median_query_seconds": 172.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 78.788, "latency": 546.998, "stderr": 3.127, "cost_per_test": 0.022555, "avg_input_tokens": 1860.4, "avg_output_tokens": 9106.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 547.0, "median_query_seconds": 370.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 78.283, "latency": 39.938, "stderr": 2.93, "cost_per_test": 0.014157, "avg_input_tokens": 1972.8, "avg_output_tokens": 6831.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 39.9, "median_query_seconds": 26.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.7": { "accuracy": 78.283, "latency": 205.538, "stderr": 2.93, "cost_per_test": 0.024294, "avg_input_tokens": 1881.3, "avg_output_tokens": 10850.3, "avg_cache_read_tokens": 1438.8, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 205.5, "median_query_seconds": 144.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 77.778, "latency": 123.747, "stderr": 2.955, "cost_per_test": 0.014168, "avg_input_tokens": 1944.6, "avg_output_tokens": 1972.4, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 123.7, "median_query_seconds": 53.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 76.768, "latency": 31.524, "stderr": 3.408, "cost_per_test": 0.001658, "avg_input_tokens": 1872.4, "avg_output_tokens": 3911.1, "avg_reasoning_tokens": 3421.5, "avg_cache_read_tokens": 9.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 31.5, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 76.768, "latency": 36.604, "stderr": 3.001, "cost_per_test": 0.16189, "avg_input_tokens": 2148.5, "avg_output_tokens": 1728.8, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 36.6, "median_query_seconds": 34.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 76.263, "latency": 41.601, "stderr": 3.088, "cost_per_test": 0.000597, "avg_input_tokens": 1817.3, "avg_output_tokens": 1282.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1785.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 41.6, "median_query_seconds": 34.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 75.758, "latency": 23.633, "stderr": 3.046, "cost_per_test": 0.002925, "avg_input_tokens": 1923.5, "avg_output_tokens": 5181.6, "avg_reasoning_tokens": 4242.6, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 23.6, "median_query_seconds": 16.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 75.758, "latency": 46.429, "stderr": 3.487, "cost_per_test": 0.005601, "avg_input_tokens": 1941.4, "avg_output_tokens": 8850.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 46.4, "median_query_seconds": 27.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 75.758, "latency": 110.423, "stderr": 3.088, "cost_per_test": 0.009533, "avg_input_tokens": 1829.7, "avg_output_tokens": 7381.6, "avg_cache_read_tokens": 31.0, "avg_cache_write_tokens": 1796.7, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 110.4, "median_query_seconds": 53.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 75.253, "latency": 139.726, "stderr": 3.067, "cost_per_test": 0.172742, "avg_input_tokens": 2148.5, "avg_output_tokens": 11086.5, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 139.7, "median_query_seconds": 129.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 74.747, "latency": 112.167, "stderr": 3.088, "cost_per_test": 0.112814, "avg_input_tokens": 2148.5, "avg_output_tokens": 7091.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 112.2, "median_query_seconds": 118.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 74.242, "latency": 91.417, "stderr": 3.108, "cost_per_test": 0.036567, "avg_input_tokens": 1873.4, "avg_output_tokens": 7842.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 91.4, "median_query_seconds": 36.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 73.737, "latency": 20.274, "stderr": 3.127, "cost_per_test": 0.083302, "avg_input_tokens": 2120.5, "avg_output_tokens": 686.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 20.3, "median_query_seconds": 19.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 73.737, "latency": 28.254, "stderr": 3.127, "cost_per_test": 0.012662, "avg_input_tokens": 1873.4, "avg_output_tokens": 2409.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 28.3, "median_query_seconds": 18.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.6": { "accuracy": 73.737, "latency": 494.128, "stderr": 3.146, "cost_per_test": 0.015371, "avg_input_tokens": 1883.3, "avg_output_tokens": 6473.3, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 494.1, "median_query_seconds": 92.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 73.232, "latency": 75.421, "stderr": 3.542, "cost_per_test": 0.050791, "avg_input_tokens": 2146.5, "avg_output_tokens": 9728.8, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 75.4, "median_query_seconds": 59.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 72.727, "latency": 35.372, "stderr": 3.165, "cost_per_test": 0.227864, "avg_input_tokens": 1873.4, "avg_output_tokens": 3329.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 35.4, "median_query_seconds": 27.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3": { "accuracy": 71.717, "latency": 25.907, "stderr": 3.201, "cost_per_test": 0.0327, "avg_input_tokens": 1781.9, "avg_output_tokens": 1823.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 25.9, "median_query_seconds": 23.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 71.212, "latency": 38.525, "stderr": 3.218, "cost_per_test": 0.083439, "avg_input_tokens": 2120.5, "avg_output_tokens": 688.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 38.5, "median_query_seconds": 37.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.5": { "accuracy": 70.707, "latency": 694.557, "stderr": 3.25, "cost_per_test": 0.020131, "avg_input_tokens": 1881.3, "avg_output_tokens": 8637.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 694.6, "median_query_seconds": 214.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 69.697, "latency": 14.615, "stderr": 3.266, "cost_per_test": 0.001837, "avg_input_tokens": 1782.9, "avg_output_tokens": 2604.5, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 14.6, "median_query_seconds": 13.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 69.697, "latency": 21.398, "stderr": 3.266, "cost_per_test": 0.000537, "avg_input_tokens": 1923.5, "avg_output_tokens": 7158.4, "avg_reasoning_tokens": 6297.9, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 21.4, "median_query_seconds": 14.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 69.192, "latency": 13.094, "stderr": 3.281, "cost_per_test": 0.016271, "avg_input_tokens": 2120.5, "avg_output_tokens": 660.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 13.1, "median_query_seconds": 11.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 69.192, "latency": 30.413, "stderr": 3.281, "cost_per_test": 0.004435, "avg_input_tokens": 1884.3, "avg_output_tokens": 850.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 30.4, "median_query_seconds": 25.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 68.687, "latency": 9.856, "stderr": 3.296, "cost_per_test": 0.015357, "avg_input_tokens": 2120.5, "avg_output_tokens": 599.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 9.9, "median_query_seconds": 9.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 68.182, "latency": 9.54, "stderr": 3.31, "cost_per_test": 0.001096, "avg_input_tokens": 1803.5, "avg_output_tokens": 794.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 9.5, "median_query_seconds": 9.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 68.182, "latency": 279.853, "stderr": 3.375, "cost_per_test": 0.005974, "avg_input_tokens": 1944.6, "avg_output_tokens": 6302.4, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 279.9, "median_query_seconds": 156.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 67.172, "latency": 4.507, "stderr": 3.337, "cost_per_test": 0.001771, "avg_input_tokens": 1901.9, "avg_output_tokens": 2780.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 4.5, "median_query_seconds": 3.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 66.162, "latency": 4.976, "stderr": 3.363, "cost_per_test": 0.000512, "avg_input_tokens": 1924.5, "avg_output_tokens": 798.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 5.0, "median_query_seconds": 4.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 65.152, "latency": 120.357, "stderr": 3.553, "cost_per_test": 0.00604, "avg_input_tokens": 1941.4, "avg_output_tokens": 19680.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 120.4, "median_query_seconds": 64.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 64.646, "latency": 8.363, "stderr": 3.397, "cost_per_test": 0.000498, "avg_input_tokens": 1942.9, "avg_output_tokens": 643.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1415.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 8.4, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 64.646, "latency": 8.974, "stderr": 3.397, "cost_per_test": 0.002537, "avg_input_tokens": 1874.4, "avg_output_tokens": 1117.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 9.0, "median_query_seconds": 7.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 64.646, "latency": 48.056, "stderr": 3.456, "cost_per_test": 0.03391, "avg_input_tokens": 1909.1, "avg_output_tokens": 13335.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 48.1, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 64.646, "latency": 66.301, "stderr": 3.419, "cost_per_test": null, "avg_input_tokens": 1909.5, "avg_output_tokens": 4163.8, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 66.3, "median_query_seconds": 52.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 64.141, "latency": 20.444, "stderr": 3.419, "cost_per_test": 0.017619, "avg_input_tokens": 1874.4, "avg_output_tokens": 1733.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 20.4, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 64.141, "latency": 21.802, "stderr": 3.324, "cost_per_test": 0.002625, "avg_input_tokens": 1891.1, "avg_output_tokens": 1119.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 21.8, "median_query_seconds": 19.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 64.141, "latency": 44.156, "stderr": 3.408, "cost_per_test": 0.004136, "avg_input_tokens": 1962.3, "avg_output_tokens": 10095.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 44.2, "median_query_seconds": 37.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 62.626, "latency": 83.642, "stderr": 3.523, "cost_per_test": 0.033256, "avg_input_tokens": 1891.8, "avg_output_tokens": 5962.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 83.6, "median_query_seconds": 56.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 62.121, "latency": 7.17, "stderr": 3.447, "cost_per_test": 0.001336, "avg_input_tokens": 1923.5, "avg_output_tokens": 2859.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 7.2, "median_query_seconds": 5.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 60.101, "latency": 45.93, "stderr": 3.494, "cost_per_test": 0.006825, "avg_input_tokens": 1891.7, "avg_output_tokens": 3942.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 45.9, "median_query_seconds": 36.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "xiaomi/mimo-v2-flash": { "accuracy": 59.091, "latency": 20.883, "stderr": 3.507, "cost_per_test": 0.001223, "avg_input_tokens": 1962.6, "avg_output_tokens": 3875.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1506.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 20.9, "median_query_seconds": 7.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 58.586, "latency": 19.603, "stderr": 3.501, "cost_per_test": 0.002135, "avg_input_tokens": 1817.3, "avg_output_tokens": 555.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 19.6, "median_query_seconds": 17.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 4.545, "latency": 233.988, "stderr": 1.48, "cost_per_test": null, "avg_input_tokens": 1977.7, "avg_output_tokens": 10101.6, "avg_reasoning_tokens": 9630.3, "avg_cache_read_tokens": 1711.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 255.1, "median_query_seconds": 246.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 56.566, "latency": 7.711, "stderr": 3.523, "cost_per_test": 0.011928, "avg_input_tokens": 2120.5, "avg_output_tokens": 371.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 7.7, "median_query_seconds": 7.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 56.061, "latency": 7.712, "stderr": 3.527, "cost_per_test": 0.004704, "avg_input_tokens": 1924.5, "avg_output_tokens": 459.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 7.7, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 54.545, "latency": 74.311, "stderr": 3.523, "cost_per_test": 0.001355, "avg_input_tokens": 2014.7, "avg_output_tokens": 11888.9, "avg_reasoning_tokens": 5693.5, "avg_cache_read_tokens": 1716.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 74.3, "median_query_seconds": 62.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 54.04, "latency": 15.717, "stderr": 3.542, "cost_per_test": 0.011827, "avg_input_tokens": 1886.4, "avg_output_tokens": 711.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 15.7, "median_query_seconds": 14.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 53.535, "latency": 19.423, "stderr": 3.544, "cost_per_test": 0.002142, "avg_input_tokens": 1817.3, "avg_output_tokens": 562.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 19.4, "median_query_seconds": 16.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-2-1212": { "accuracy": 52.02, "latency": 20.466, "stderr": 3.55, "cost_per_test": 0.011263, "avg_input_tokens": 1880.4, "avg_output_tokens": 750.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 20.5, "median_query_seconds": 13.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 51.01, "latency": 4.255, "stderr": 3.438, "cost_per_test": 0.000576, "avg_input_tokens": 1874.4, "avg_output_tokens": 970.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 4.3, "median_query_seconds": 3.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-2024-05-13": { "accuracy": 50.505, "latency": 10.468, "stderr": 3.553, "cost_per_test": 0.019577, "avg_input_tokens": 1874.4, "avg_output_tokens": 680.3, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 10.5, "median_query_seconds": 9.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 50.0, "latency": 11.887, "stderr": 3.363, "cost_per_test": 0.012408, "avg_input_tokens": 1965.3, "avg_output_tokens": 749.5, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 11.9, "median_query_seconds": 10.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 48.485, "latency": 7.474, "stderr": 3.552, "cost_per_test": 0.002331, "avg_input_tokens": 1916.5, "avg_output_tokens": 732.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 7.5, "median_query_seconds": 7.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 46.97, "latency": 16.517, "stderr": 3.547, "cost_per_test": 0.008783, "avg_input_tokens": 2219.1, "avg_output_tokens": 724.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 16.5, "median_query_seconds": 12.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 44.949, "latency": 3.47, "stderr": 3.535, "cost_per_test": 0.000299, "avg_input_tokens": 1924.5, "avg_output_tokens": 514.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 3.5, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 44.949, "latency": 8.344, "stderr": 3.535, "cost_per_test": 0.000845, "avg_input_tokens": 1892.1, "avg_output_tokens": 777.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 8.3, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 44.949, "latency": 15.039, "stderr": 3.535, "cost_per_test": 0.00067, "avg_input_tokens": 1874.4, "avg_output_tokens": 648.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 15.0, "median_query_seconds": 12.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 44.444, "latency": 16.209, "stderr": 3.456, "cost_per_test": null, "avg_input_tokens": 1913.4, "avg_output_tokens": 1049.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 16.2, "median_query_seconds": 14.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 43.939, "latency": 10.003, "stderr": 3.523, "cost_per_test": 0.000366, "avg_input_tokens": 1892.1, "avg_output_tokens": 745.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 10.0, "median_query_seconds": 6.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 41.414, "latency": 14.853, "stderr": 3.363, "cost_per_test": 0.008222, "avg_input_tokens": 2043.8, "avg_output_tokens": 516.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 14.9, "median_query_seconds": 13.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 40.909, "latency": 10.759, "stderr": 3.494, "cost_per_test": 0.000746, "avg_input_tokens": 1803.5, "avg_output_tokens": 714.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 10.8, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 36.364, "latency": 6.978, "stderr": 3.419, "cost_per_test": 0.003004, "avg_input_tokens": 2120.5, "avg_output_tokens": 326.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 7.0, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 34.848, "latency": 248.163, "stderr": 3.386, "cost_per_test": 0.089386, "avg_input_tokens": 1889.3, "avg_output_tokens": 11540.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 91, "avg_turns": 1.0, "avg_query_seconds": 248.2, "median_query_seconds": 237.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 33.333, "latency": 4.791, "stderr": 3.201, "cost_per_test": 0.000652, "avg_input_tokens": 2043.8, "avg_output_tokens": 608.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 4.8, "median_query_seconds": 4.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "openai/gpt-3.5-turbo": { "accuracy": 28.788, "latency": 2.834, "stderr": 3.201, "cost_per_test": 0.001296, "avg_input_tokens": 1890.6, "avg_output_tokens": 233.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 2.8, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "cohere/command-r-plus": { "accuracy": 28.283, "latency": 8.0, "stderr": 3.183, "cost_per_test": 0.009178, "avg_input_tokens": 1965.3, "avg_output_tokens": 426.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 8.0, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null } }, "diamond_zero_shot_cot": { "openai/gpt-5.6-sol": { "accuracy": 95.455, "latency": 44.206, "stderr": 1.48, "cost_per_test": 0.084288, "avg_input_tokens": 637.8, "avg_output_tokens": 2703.3, "avg_reasoning_tokens": 2666.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 44.2, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 95.455, "latency": 206.939, "stderr": 1.48, "cost_per_test": 0.044383, "avg_input_tokens": 278.7, "avg_output_tokens": 7304.7, "avg_reasoning_tokens": 7046.6, "avg_cache_read_tokens": 1.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 206.9, "median_query_seconds": 92.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 94.949, "latency": 64.024, "stderr": 1.556, "cost_per_test": 0.077857, "avg_input_tokens": 256.1, "avg_output_tokens": 6445.4, "avg_reasoning_tokens": 5741.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 64.0, "median_query_seconds": 40.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 94.949, "latency": 267.316, "stderr": 1.556, "cost_per_test": 0.076987, "avg_input_tokens": 311.0, "avg_output_tokens": 12727.5, "avg_reasoning_tokens": 12437.1, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 267.3, "median_query_seconds": 148.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 94.444, "latency": 19.256, "stderr": 1.76, "cost_per_test": 0.032582, "avg_input_tokens": 256.1, "avg_output_tokens": 4293.0, "avg_reasoning_tokens": 3698.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 19.3, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 94.444, "latency": 40.592, "stderr": 1.628, "cost_per_test": 0.090786, "avg_input_tokens": 373.5, "avg_output_tokens": 3552.1, "avg_reasoning_tokens": 2861.2, "avg_cache_write_tokens": 93.9, "turn_metrics": { "tasks_measured": 196, "avg_turns": 1.0, "avg_query_seconds": 40.6, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 93.939, "latency": 18.517, "stderr": 2.277, "cost_per_test": 0.051126, "avg_input_tokens": 256.1, "avg_output_tokens": 6765.6, "avg_reasoning_tokens": 6160.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 18.5, "median_query_seconds": 9.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 93.939, "latency": 28.557, "stderr": 2.142, "cost_per_test": 0.061459, "avg_input_tokens": 256.1, "avg_output_tokens": 5078.9, "avg_reasoning_tokens": 4429.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 28.6, "median_query_seconds": 18.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 93.939, "latency": 44.083, "stderr": 3.25, "cost_per_test": 0.184706, "avg_input_tokens": 384.2, "avg_output_tokens": 3612.3, "avg_cache_write_tokens": 98.4, "turn_metrics": { "tasks_measured": 197, "avg_turns": 1.0, "avg_query_seconds": 44.1, "median_query_seconds": 15.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 93.434, "latency": 67.496, "stderr": 3.296, "cost_per_test": 0.290028, "avg_input_tokens": 376.5, "avg_output_tokens": 5753.4, "avg_cache_write_tokens": 24.9, "turn_metrics": { "tasks_measured": 197, "avg_turns": 1.0, "avg_query_seconds": 67.5, "median_query_seconds": 21.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 92.929, "latency": 106.252, "stderr": 1.822, "cost_per_test": 0.073175, "avg_input_tokens": 1089.5, "avg_output_tokens": 3883.7, "avg_reasoning_tokens": 3803.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 106.3, "median_query_seconds": 49.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 92.929, "latency": 136.373, "stderr": 3.25, "cost_per_test": 0.340421, "avg_input_tokens": 371.6, "avg_output_tokens": 13611.1, "avg_cache_write_tokens": 25.1, "turn_metrics": { "tasks_measured": 195, "avg_turns": 1.0, "avg_query_seconds": 136.4, "median_query_seconds": 44.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 92.929, "latency": 147.589, "stderr": 1.822, "cost_per_test": 0.041673, "avg_input_tokens": 278.7, "avg_output_tokens": 6852.6, "avg_reasoning_tokens": 6502.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 147.6, "median_query_seconds": 104.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 92.929, "latency": 280.541, "stderr": 2.68, "cost_per_test": 0.069677, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 92.929, "latency": 313.788, "stderr": 2.188, "cost_per_test": 0.028001, "avg_input_tokens": 415.7, "avg_output_tokens": 11592.1, "avg_cache_read_tokens": 144.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 313.8, "median_query_seconds": 67.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 92.424, "latency": 8.36, "stderr": 2.4, "cost_per_test": 0.02089, "avg_input_tokens": 256.1, "avg_output_tokens": 2734.1, "avg_reasoning_tokens": 2217.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 8.4, "median_query_seconds": 5.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 92.424, "latency": 63.916, "stderr": 1.881, "cost_per_test": 0.091397, "avg_input_tokens": 256.1, "avg_output_tokens": 7573.7, "avg_reasoning_tokens": 6678.6, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 63.9, "median_query_seconds": 56.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 92.424, "latency": 115.375, "stderr": 1.937, "cost_per_test": 0.083665, "avg_input_tokens": 339.2, "avg_output_tokens": 5564.8, "avg_reasoning_tokens": 5362.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 115.4, "median_query_seconds": 47.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 91.919, "latency": 30.09, "stderr": 2.68, "cost_per_test": 0.034726, "avg_input_tokens": 259.4, "avg_output_tokens": 2850.6, "avg_reasoning_tokens": 2753.5, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 30.1, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 91.919, "latency": 82.463, "stderr": 3.296, "cost_per_test": 0.037824, "avg_input_tokens": 404.5, "avg_output_tokens": 8818.1, "avg_reasoning_tokens": 8358.2, "avg_cache_read_tokens": 144.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 82.5, "median_query_seconds": 40.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 91.919, "latency": 218.997, "stderr": 2.188, "cost_per_test": 0.222481, "avg_input_tokens": 259.4, "avg_output_tokens": 7508.3, "avg_reasoning_tokens": 7280.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 219.0, "median_query_seconds": 80.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 91.414, "latency": 176.161, "stderr": 2.68, "cost_per_test": 0.149003, "avg_input_tokens": 1423.1, "avg_output_tokens": 21744.9, "avg_reasoning_tokens": 11279.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 176.2, "median_query_seconds": 41.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 90.909, "latency": 61.485, "stderr": 2.583, "cost_per_test": 0.011853, "avg_input_tokens": 542.5, "avg_output_tokens": 9786.9, "avg_reasoning_tokens": 9636.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 61.5, "median_query_seconds": 26.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 90.909, "latency": 94.785, "stderr": 2.093, "cost_per_test": 0.002959, "avg_input_tokens": 302.5, "avg_output_tokens": 4856.6, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 94.8, "median_query_seconds": 63.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 90.909, "latency": 300.77, "stderr": 2.093, "cost_per_test": 0.278848, "avg_input_tokens": 311.5, "avg_output_tokens": 18527.6, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 300.8, "median_query_seconds": 99.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 90.404, "latency": 71.088, "stderr": 2.799, "cost_per_test": 0.070514, "avg_input_tokens": 377.1, "avg_output_tokens": 4623.9, "avg_cache_write_tokens": 32.0, "turn_metrics": { "tasks_measured": 196, "avg_turns": 1.0, "avg_query_seconds": 71.1, "median_query_seconds": 21.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 90.404, "latency": 170.513, "stderr": 2.093, "cost_per_test": 0.078189, "avg_input_tokens": 273.0, "avg_output_tokens": 10334.2, "avg_reasoning_tokens": 9740.3, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 170.5, "median_query_seconds": 63.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 89.899, "latency": 58.717, "stderr": 2.142, "cost_per_test": 0.033188, "avg_input_tokens": 362.3, "avg_output_tokens": 5445.4, "avg_reasoning_tokens": 5022.8, "avg_cache_read_tokens": 116.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 58.7, "median_query_seconds": 43.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 89.899, "latency": 79.801, "stderr": 2.548, "cost_per_test": 0.012785, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.3": { "accuracy": 89.899, "latency": 1082.674, "stderr": 2.142, "cost_per_test": 0.026645, "avg_input_tokens": 250.9, "avg_output_tokens": 10533.3, "avg_reasoning_tokens": 10164.7, "avg_cache_read_tokens": 2.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 1082.7, "median_query_seconds": 642.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 89.394, "latency": 82.513, "stderr": 2.188, "cost_per_test": 0.043352, "avg_input_tokens": 256.1, "avg_output_tokens": 14408.1, "avg_reasoning_tokens": 13748.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 82.5, "median_query_seconds": 80.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 89.394, "latency": 201.167, "stderr": 2.319, "cost_per_test": 0.377529, "avg_input_tokens": 375.9, "avg_output_tokens": 16365.3, "avg_cache_write_tokens": 25.9, "turn_metrics": { "tasks_measured": 189, "avg_turns": 1.0, "avg_query_seconds": 201.2, "median_query_seconds": 86.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 88.889, "latency": 129.642, "stderr": 2.233, "cost_per_test": 0.048475, "avg_input_tokens": 315.0, "avg_output_tokens": 16105.8, "avg_reasoning_tokens": 15808.7, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 129.6, "median_query_seconds": 73.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-0709": { "accuracy": 88.384, "latency": 111.554, "stderr": 2.277, "cost_per_test": 0.143643, "avg_input_tokens": 244.9, "avg_output_tokens": 9527.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 111.6, "median_query_seconds": 82.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 88.384, "latency": 589.087, "stderr": 3.281, "cost_per_test": 0.084991, "avg_input_tokens": 261.2, "avg_output_tokens": 21185.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 589.1, "median_query_seconds": 349.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 88.384, "latency": 977.721, "stderr": 2.4, "cost_per_test": 0.138778, "avg_input_tokens": 328.3, "avg_output_tokens": 34952.5, "avg_reasoning_tokens": 34662.9, "avg_cache_read_tokens": 53.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 977.7, "median_query_seconds": 595.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 87.879, "latency": 198.108, "stderr": 2.583, "cost_per_test": 0.015282, "avg_input_tokens": 260.7, "avg_output_tokens": 12675.2, "avg_cache_read_tokens": 74.9, "avg_cache_write_tokens": 185.8, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 198.1, "median_query_seconds": 104.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5.2": { "accuracy": 87.879, "latency": 271.144, "stderr": 2.36, "cost_per_test": 0.06444, "avg_input_tokens": 267.2, "avg_output_tokens": 14560.5, "avg_reasoning_tokens": 14025.1, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 271.1, "median_query_seconds": 117.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 87.374, "latency": 144.698, "stderr": 3.201, "cost_per_test": 0.001659, "avg_input_tokens": 267.2, "avg_output_tokens": 6557.2, "avg_reasoning_tokens": 6174.0, "avg_cache_read_tokens": 3.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 144.7, "median_query_seconds": 51.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3": { "accuracy": 87.374, "latency": 216.036, "stderr": 2.853, "cost_per_test": 0.052834, "avg_input_tokens": 267.2, "avg_output_tokens": 11922.8, "avg_reasoning_tokens": 11592.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 216.0, "median_query_seconds": 75.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 87.374, "latency": 278.169, "stderr": 2.853, "cost_per_test": 0.021457, "avg_input_tokens": 267.4, "avg_output_tokens": 17833.8, "avg_cache_read_tokens": 99.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 278.2, "median_query_seconds": 226.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 86.869, "latency": 88.114, "stderr": 2.4, "cost_per_test": 0.072084, "avg_input_tokens": 259.4, "avg_output_tokens": 7176.0, "avg_reasoning_tokens": 6889.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 88.1, "median_query_seconds": 40.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 86.869, "latency": 88.527, "stderr": 2.4, "cost_per_test": 0.019122, "avg_input_tokens": 273.0, "avg_output_tokens": 7922.1, "avg_reasoning_tokens": 7046.4, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 88.5, "median_query_seconds": 81.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 86.869, "latency": 384.478, "stderr": 2.4, "cost_per_test": 0.075911, "avg_input_tokens": 270.4, "avg_output_tokens": 12597.7, "avg_reasoning_tokens": 11304.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 197, "avg_turns": 1.0, "avg_query_seconds": 384.5, "median_query_seconds": 308.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 86.364, "latency": 20.422, "stderr": 2.476, "cost_per_test": 0.002352, "avg_input_tokens": 354.9, "avg_output_tokens": 4562.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 20.4, "median_query_seconds": 12.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 86.364, "latency": 145.221, "stderr": 2.439, "cost_per_test": 0.02817, "avg_input_tokens": 273.0, "avg_output_tokens": 9344.5, "avg_reasoning_tokens": 8661.1, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 145.2, "median_query_seconds": 86.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 86.364, "latency": 147.729, "stderr": 2.439, "cost_per_test": 0.071283, "avg_input_tokens": 259.4, "avg_output_tokens": 7095.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 147.7, "median_query_seconds": 72.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 86.364, "latency": 335.213, "stderr": 2.711, "cost_per_test": 0.179663, "avg_input_tokens": 267.4, "avg_output_tokens": 19114.3, "avg_cache_read_tokens": 82.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 335.2, "median_query_seconds": 281.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 85.859, "latency": 13.756, "stderr": 2.476, "cost_per_test": 0.012083, "avg_input_tokens": 256.1, "avg_output_tokens": 4802.5, "avg_reasoning_tokens": 4284.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 13.8, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 85.859, "latency": 174.88, "stderr": 2.476, "cost_per_test": null, "avg_input_tokens": 269.1, "avg_output_tokens": 16050.9, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 174.9, "median_query_seconds": 84.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 85.354, "latency": 57.231, "stderr": 2.583, "cost_per_test": 0.033212, "avg_input_tokens": 259.4, "avg_output_tokens": 4086.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 57.2, "median_query_seconds": 30.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 85.354, "latency": 137.082, "stderr": 3.501, "cost_per_test": 0.691649, "avg_input_tokens": 323.5, "avg_output_tokens": 9157.3, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 137.1, "median_query_seconds": 116.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 85.354, "latency": 181.68, "stderr": 3.324, "cost_per_test": 0.035118, "avg_input_tokens": 260.0, "avg_output_tokens": 11957.4, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 181.7, "median_query_seconds": 118.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 85.354, "latency": 391.171, "stderr": 3.408, "cost_per_test": 0.418665, "avg_input_tokens": 309.9, "avg_output_tokens": 27848.2, "avg_cache_write_tokens": 16.2, "turn_metrics": { "tasks_measured": 197, "avg_turns": 1.0, "avg_query_seconds": 391.2, "median_query_seconds": 135.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 84.848, "latency": 66.823, "stderr": 2.616, "cost_per_test": 0.002743, "avg_input_tokens": 393.9, "avg_output_tokens": 5377.5, "avg_reasoning_tokens": 5042.9, "avg_cache_read_tokens": 161.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 66.8, "median_query_seconds": 39.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 84.343, "latency": 48.704, "stderr": 2.905, "cost_per_test": 0.003854, "avg_input_tokens": 277.6, "avg_output_tokens": 17424.7, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 48.7, "median_query_seconds": 16.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 84.343, "latency": 265.025, "stderr": 3.55, "cost_per_test": 0.006663, "avg_input_tokens": 508.0, "avg_output_tokens": 7739.6, "avg_reasoning_tokens": 7321.6, "avg_cache_read_tokens": 192.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 265.0, "median_query_seconds": 109.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5-thinking": { "accuracy": 83.838, "latency": 250.83, "stderr": 2.616, "cost_per_test": 0.036218, "avg_input_tokens": 244.3, "avg_output_tokens": 11416.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 250.8, "median_query_seconds": 180.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 83.333, "latency": 82.449, "stderr": 2.649, "cost_per_test": 0.003533, "avg_input_tokens": 273.0, "avg_output_tokens": 8764.4, "avg_reasoning_tokens": 7827.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 82.4, "median_query_seconds": 72.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 83.333, "latency": 101.368, "stderr": 2.905, "cost_per_test": 0.002781, "avg_input_tokens": 506.6, "avg_output_tokens": 9774.0, "avg_reasoning_tokens": 9570.8, "avg_cache_read_tokens": 192.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 101.4, "median_query_seconds": 61.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.1": { "accuracy": 83.249, "latency": 301.653, "stderr": 2.661, "cost_per_test": 0.054832, "avg_input_tokens": 260.4, "avg_output_tokens": 17054.2, "avg_reasoning_tokens": 16745.3, "avg_cache_read_tokens": 1.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 197, "avg_turns": 1.0, "avg_query_seconds": 301.7, "median_query_seconds": 190.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 82.828, "latency": 16.842, "stderr": 2.68, "cost_per_test": 0.002266, "avg_input_tokens": 256.1, "avg_output_tokens": 5600.9, "avg_reasoning_tokens": 4953.5, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 16.8, "median_query_seconds": 11.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 82.323, "latency": 94.328, "stderr": 2.711, "cost_per_test": 0.013029, "avg_input_tokens": 259.4, "avg_output_tokens": 6482.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 94.3, "median_query_seconds": 106.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.7": { "accuracy": 81.818, "latency": 216.62, "stderr": 2.741, "cost_per_test": 0.025437, "avg_input_tokens": 260.3, "avg_output_tokens": 11493.8, "avg_cache_read_tokens": 10.4, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 216.6, "median_query_seconds": 161.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 81.818, "latency": 274.231, "stderr": 2.741, "cost_per_test": 0.003664, "avg_input_tokens": 248.8, "avg_output_tokens": 8866.8, "avg_reasoning_tokens": 8687.3, "avg_cache_read_tokens": 215.8, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 274.2, "median_query_seconds": 213.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 81.633, "latency": 126.056, "stderr": 3.001, "cost_per_test": 0.119852, "avg_input_tokens": 320.9, "avg_output_tokens": 7926.0, "turn_metrics": { "tasks_measured": 196, "avg_turns": 1.0, "avg_query_seconds": 126.1, "median_query_seconds": 128.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 81.313, "latency": 42.429, "stderr": 2.799, "cost_per_test": 0.011682, "avg_input_tokens": 256.5, "avg_output_tokens": 1136.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 42.4, "median_query_seconds": 29.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 81.313, "latency": 65.923, "stderr": 2.853, "cost_per_test": 0.006824, "avg_input_tokens": 327.4, "avg_output_tokens": 11290.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 65.9, "median_query_seconds": 38.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 81.313, "latency": 66.141, "stderr": 3.397, "cost_per_test": 0.013209, "avg_input_tokens": 259.4, "avg_output_tokens": 6572.1, "avg_reasoning_tokens": 6496.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 66.1, "median_query_seconds": 31.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 81.313, "latency": 160.401, "stderr": 2.93, "cost_per_test": 0.013312, "avg_input_tokens": 277.7, "avg_output_tokens": 11015.6, "avg_cache_read_tokens": 31.0, "avg_cache_write_tokens": 244.7, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 160.4, "median_query_seconds": 97.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 80.808, "latency": 19.137, "stderr": 2.93, "cost_per_test": 0.002833, "avg_input_tokens": 256.5, "avg_output_tokens": 5187.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 19.1, "median_query_seconds": 12.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max": { "accuracy": 80.303, "latency": 71.669, "stderr": 2.826, "cost_per_test": 0.015765, "avg_input_tokens": 267.6, "avg_output_tokens": 2573.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 71.7, "median_query_seconds": 49.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 79.798, "latency": 12.625, "stderr": 3.472, "cost_per_test": 0.056963, "avg_input_tokens": 294.5, "avg_output_tokens": 700.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 12.6, "median_query_seconds": 12.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 79.798, "latency": 180.634, "stderr": 3.001, "cost_per_test": 0.018424, "avg_input_tokens": 280.7, "avg_output_tokens": 7638.2, "avg_cache_read_tokens": 36.0, "avg_cache_write_tokens": 244.7, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 180.6, "median_query_seconds": 93.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 78.283, "latency": 27.473, "stderr": 3.429, "cost_per_test": 0.001472, "avg_input_tokens": 259.9, "avg_output_tokens": 3647.0, "avg_reasoning_tokens": 3394.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 27.5, "median_query_seconds": 13.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 78.283, "latency": 40.979, "stderr": 2.955, "cost_per_test": 0.003637, "avg_input_tokens": 244.9, "avg_output_tokens": 7128.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 41.0, "median_query_seconds": 34.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 78.283, "latency": 624.112, "stderr": 3.046, "cost_per_test": 0.023588, "avg_input_tokens": 258.9, "avg_output_tokens": 9822.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 624.1, "median_query_seconds": 434.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 77.778, "latency": 89.82, "stderr": 2.955, "cost_per_test": 0.012593, "avg_input_tokens": 267.6, "avg_output_tokens": 2045.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 89.8, "median_query_seconds": 45.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 77.273, "latency": 23.427, "stderr": 3.046, "cost_per_test": 0.002741, "avg_input_tokens": 256.5, "avg_output_tokens": 5221.3, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 23.4, "median_query_seconds": 16.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-3": { "accuracy": 76.768, "latency": 33.917, "stderr": 3.046, "cost_per_test": 0.045785, "avg_input_tokens": 243.9, "avg_output_tokens": 3003.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 33.9, "median_query_seconds": 31.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 76.768, "latency": 108.55, "stderr": 3.046, "cost_per_test": 0.035147, "avg_input_tokens": 259.4, "avg_output_tokens": 7923.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 108.6, "median_query_seconds": 40.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 76.263, "latency": 13.854, "stderr": 3.046, "cost_per_test": 0.001361, "avg_input_tokens": 244.9, "avg_output_tokens": 2574.3, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 13.9, "median_query_seconds": 12.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 76.263, "latency": 52.056, "stderr": 3.024, "cost_per_test": 0.000703, "avg_input_tokens": 249.3, "avg_output_tokens": 1636.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 216.6, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 52.1, "median_query_seconds": 45.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 75.758, "latency": 34.939, "stderr": 3.108, "cost_per_test": 0.127465, "avg_input_tokens": 322.5, "avg_output_tokens": 1635.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 34.9, "median_query_seconds": 30.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 75.253, "latency": 31.754, "stderr": 3.067, "cost_per_test": 0.01326, "avg_input_tokens": 259.4, "avg_output_tokens": 2948.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 31.8, "median_query_seconds": 17.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 75.253, "latency": 124.748, "stderr": 3.108, "cost_per_test": 0.120597, "avg_input_tokens": 322.5, "avg_output_tokens": 7975.3, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 124.7, "median_query_seconds": 133.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 75.253, "latency": 170.315, "stderr": 3.067, "cost_per_test": 0.204759, "avg_input_tokens": 322.5, "avg_output_tokens": 13586.1, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 170.3, "median_query_seconds": 169.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.6": { "accuracy": 75.253, "latency": 455.539, "stderr": 3.067, "cost_per_test": 0.015448, "avg_input_tokens": 261.4, "avg_output_tokens": 6950.5, "turn_metrics": { "tasks_measured": 197, "avg_turns": 1.0, "avg_query_seconds": 455.5, "median_query_seconds": 85.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 73.737, "latency": 44.979, "stderr": 3.146, "cost_per_test": 0.277401, "avg_input_tokens": 259.4, "avg_output_tokens": 4558.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 45.0, "median_query_seconds": 30.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 73.737, "latency": 52.454, "stderr": 3.127, "cost_per_test": 0.004527, "avg_input_tokens": 270.2, "avg_output_tokens": 1418.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 52.5, "median_query_seconds": 39.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-4.5": { "accuracy": 73.737, "latency": 713.892, "stderr": 3.146, "cost_per_test": 0.018777, "avg_input_tokens": 260.3, "avg_output_tokens": 8463.9, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 713.9, "median_query_seconds": 220.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 72.727, "latency": 20.631, "stderr": 3.386, "cost_per_test": 0.001675, "avg_input_tokens": 256.1, "avg_output_tokens": 1031.3, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 20.6, "median_query_seconds": 18.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 72.727, "latency": 169.828, "stderr": 3.419, "cost_per_test": 0.007436, "avg_input_tokens": 327.4, "avg_output_tokens": 24711.3, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 169.8, "median_query_seconds": 105.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 72.222, "latency": 332.583, "stderr": 3.337, "cost_per_test": 0.006971, "avg_input_tokens": 267.6, "avg_output_tokens": 7855.1, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 332.6, "median_query_seconds": 187.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 71.212, "latency": 9.589, "stderr": 3.25, "cost_per_test": 0.002112, "avg_input_tokens": 260.4, "avg_output_tokens": 1255.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 9.6, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 71.212, "latency": 90.985, "stderr": 3.456, "cost_per_test": 0.059337, "avg_input_tokens": 322.5, "avg_output_tokens": 11802.9, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 91.0, "median_query_seconds": 72.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 70.707, "latency": 10.602, "stderr": 3.337, "cost_per_test": 0.000831, "avg_input_tokens": 250.5, "avg_output_tokens": 881.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 10.6, "median_query_seconds": 9.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 70.707, "latency": 19.822, "stderr": 3.234, "cost_per_test": 0.000425, "avg_input_tokens": 256.5, "avg_output_tokens": 5979.5, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 19.8, "median_query_seconds": 12.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 69.697, "latency": 12.541, "stderr": 3.266, "cost_per_test": 0.010833, "avg_input_tokens": 294.5, "avg_output_tokens": 663.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 12.5, "median_query_seconds": 11.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 69.697, "latency": 17.986, "stderr": 3.266, "cost_per_test": 0.050314, "avg_input_tokens": 294.5, "avg_output_tokens": 612.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 18.0, "median_query_seconds": 17.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 68.687, "latency": 33.342, "stderr": 3.296, "cost_per_test": 0.051403, "avg_input_tokens": 294.5, "avg_output_tokens": 626.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 33.3, "median_query_seconds": 33.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 66.667, "latency": 25.835, "stderr": 3.375, "cost_per_test": 0.010086, "avg_input_tokens": 260.4, "avg_output_tokens": 1195.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 25.8, "median_query_seconds": 7.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 66.162, "latency": 7.377, "stderr": 3.375, "cost_per_test": 0.001135, "avg_input_tokens": 256.5, "avg_output_tokens": 2772.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 7.4, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 66.162, "latency": 8.579, "stderr": 3.363, "cost_per_test": 0.008588, "avg_input_tokens": 294.5, "avg_output_tokens": 513.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 8.6, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 65.657, "latency": 8.956, "stderr": 3.386, "cost_per_test": 0.000392, "avg_input_tokens": 405.9, "avg_output_tokens": 671.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 169.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 9.0, "median_query_seconds": 6.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 64.646, "latency": 27.347, "stderr": 3.419, "cost_per_test": 0.000969, "avg_input_tokens": 248.3, "avg_output_tokens": 828.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 27.3, "median_query_seconds": 23.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 64.141, "latency": 6.63, "stderr": 3.408, "cost_per_test": 0.000469, "avg_input_tokens": 256.5, "avg_output_tokens": 1108.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 6.6, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 62.626, "latency": 55.277, "stderr": 3.535, "cost_per_test": 0.00502, "avg_input_tokens": 330.0, "avg_output_tokens": 12508.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 55.3, "median_query_seconds": 45.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 62.121, "latency": 6.771, "stderr": 3.456, "cost_per_test": 0.006229, "avg_input_tokens": 294.5, "avg_output_tokens": 356.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 6.8, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 62.121, "latency": 104.648, "stderr": 3.363, "cost_per_test": 0.038764, "avg_input_tokens": 256.1, "avg_output_tokens": 7650.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 104.6, "median_query_seconds": 72.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 60.606, "latency": 7.706, "stderr": 3.472, "cost_per_test": 0.002606, "avg_input_tokens": 256.5, "avg_output_tokens": 457.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 7.7, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 60.101, "latency": 88.877, "stderr": 3.512, "cost_per_test": null, "avg_input_tokens": 283.5, "avg_output_tokens": 5428.2, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 88.9, "median_query_seconds": 77.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "xiaomi/mimo-v2-flash": { "accuracy": 59.596, "latency": 20.921, "stderr": 3.553, "cost_per_test": 0.0012, "avg_input_tokens": 285.6, "avg_output_tokens": 3926.3, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 69.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 20.9, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 57.071, "latency": 2.828, "stderr": 3.531, "cost_per_test": 0.000426, "avg_input_tokens": 364.9, "avg_output_tokens": 705.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 2.8, "median_query_seconds": 2.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 56.566, "latency": 7.629, "stderr": 3.523, "cost_per_test": 0.000326, "avg_input_tokens": 256.5, "avg_output_tokens": 458.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 7.6, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 56.566, "latency": 46.573, "stderr": 3.539, "cost_per_test": 0.007291, "avg_input_tokens": 256.1, "avg_output_tokens": 4775.2, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 46.6, "median_query_seconds": 41.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 55.556, "latency": 23.141, "stderr": 3.539, "cost_per_test": 0.000795, "avg_input_tokens": 248.3, "avg_output_tokens": 634.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 23.1, "median_query_seconds": 19.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 55.556, "latency": 64.312, "stderr": 3.544, "cost_per_test": 0.001391, "avg_input_tokens": 307.7, "avg_output_tokens": 6813.9, "avg_reasoning_tokens": 6410.8, "avg_cache_read_tokens": 47.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 64.3, "median_query_seconds": 53.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 53.535, "latency": 14.131, "stderr": 3.55, "cost_per_test": 0.007103, "avg_input_tokens": 272.4, "avg_output_tokens": 642.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 14.1, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 53.03, "latency": 13.424, "stderr": 3.55, "cost_per_test": 0.000563, "avg_input_tokens": 250.5, "avg_output_tokens": 878.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 13.4, "median_query_seconds": 12.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 51.515, "latency": 7.211, "stderr": 3.552, "cost_per_test": 0.000886, "avg_input_tokens": 290.5, "avg_output_tokens": 715.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 7.2, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 51.01, "latency": 16.755, "stderr": 3.523, "cost_per_test": null, "avg_input_tokens": 286.4, "avg_output_tokens": 1099.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 16.8, "median_query_seconds": 15.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 49.495, "latency": 421.926, "stderr": 3.547, "cost_per_test": null, "avg_input_tokens": 270.7, "avg_output_tokens": 22972.3, "avg_reasoning_tokens": 11282.0, "avg_cache_read_tokens": 37.4, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 278.8, "median_query_seconds": 264.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 50.505, "latency": 4.128, "stderr": 3.512, "cost_per_test": 0.0004, "avg_input_tokens": 260.4, "avg_output_tokens": 934.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 4.1, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 50.505, "latency": 38.848, "stderr": 3.363, "cost_per_test": 0.026324, "avg_input_tokens": 255.1, "avg_output_tokens": 10498.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 38.8, "median_query_seconds": 8.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4o-2024-05-13": { "accuracy": 50.0, "latency": 9.688, "stderr": 3.553, "cost_per_test": 0.011302, "avg_input_tokens": 260.4, "avg_output_tokens": 666.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 9.7, "median_query_seconds": 8.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-2-1212": { "accuracy": 49.495, "latency": 19.89, "stderr": 3.55, "cost_per_test": 0.007736, "avg_input_tokens": 253.4, "avg_output_tokens": 722.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 19.9, "median_query_seconds": 12.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 48.485, "latency": 15.087, "stderr": 3.523, "cost_per_test": 0.004644, "avg_input_tokens": 293.1, "avg_output_tokens": 676.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 15.1, "median_query_seconds": 11.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 46.97, "latency": 3.311, "stderr": 3.547, "cost_per_test": 0.000167, "avg_input_tokens": 256.5, "avg_output_tokens": 491.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 3.3, "median_query_seconds": 3.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 46.97, "latency": 11.092, "stderr": 3.067, "cost_per_test": 0.00778, "avg_input_tokens": 269.3, "avg_output_tokens": 710.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 11.1, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 44.444, "latency": 7.645, "stderr": 3.472, "cost_per_test": 0.000191, "avg_input_tokens": 256.1, "avg_output_tokens": 574.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 7.6, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 43.434, "latency": 15.6, "stderr": 3.523, "cost_per_test": 0.000403, "avg_input_tokens": 260.4, "avg_output_tokens": 607.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 15.6, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 39.394, "latency": 6.143, "stderr": 3.472, "cost_per_test": 0.001437, "avg_input_tokens": 294.5, "avg_output_tokens": 300.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 6.1, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 34.848, "latency": 264.176, "stderr": 3.375, "cost_per_test": 0.101592, "avg_input_tokens": 252.8, "avg_output_tokens": 13495.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 85, "avg_turns": 1.0, "avg_query_seconds": 264.2, "median_query_seconds": 240.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-r-plus": { "accuracy": 33.838, "latency": 8.387, "stderr": 3.266, "cost_per_test": 0.005699, "avg_input_tokens": 269.3, "avg_output_tokens": 502.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 8.4, "median_query_seconds": 7.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "openai/gpt-3.5-turbo": { "accuracy": 32.323, "latency": 2.423, "stderr": 3.266, "cost_per_test": 0.000427, "avg_input_tokens": 262.6, "avg_output_tokens": 196.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 2.4, "median_query_seconds": 1.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 30.808, "latency": 5.115, "stderr": 2.36, "cost_per_test": 0.000322, "avg_input_tokens": 285.8, "avg_output_tokens": 662.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 5.1, "median_query_seconds": 4.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 30.808, "latency": 16.089, "stderr": 2.513, "cost_per_test": 0.005158, "avg_input_tokens": 285.8, "avg_output_tokens": 573.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 198, "avg_turns": 1.0, "avg_query_seconds": 16.1, "median_query_seconds": 15.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } } } }