{ "metadata": { "benchmark": "MMLU Pro", "slug": "mmlu_pro", "description": "Academic multiple-choice benchmark covering 14 subjects including STEM, humanities, and social sciences.", "benchmark_id": "mmlu_pro", "family": "mmlu_pro", "version": "1", "updated": "2026-09-01", "dataset_type": "public", "industry": "academic", "tasks": { "overall": "Overall", "biology": "Biology", "business": "Business", "chemistry": "Chemistry", "computer_science": "Computer Science", "economics": "Economics", "engineering": "Engineering", "health": "Health", "history": "History", "law": "Law", "math": "Math", "other": "Others", "philosophy": "Philosophy", "physics": "Physics", "psychology": "Psychology" }, "models": [ "ai21labs/jamba-large-1.6", "ai21labs/jamba-mini-1.6", "alibaba/qwen3-max", "alibaba/qwen3-max-2026-01-23", "alibaba/qwen3-max-preview", "alibaba/qwen3.5-flash", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "ant/ling-3.0-flash-2607", "anthropic/claude-3-5-haiku-20241022", "anthropic/claude-3-5-sonnet-20241022", "anthropic/claude-3-7-sonnet-20250219", "anthropic/claude-3-7-sonnet-20250219-thinking", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-1-20250805-thinking", "anthropic/claude-opus-4-20250514", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "cohere/command-a-03-2025", "cohere/command-r-plus", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "fireworks/deepseek-r1", "fireworks/deepseek-v3", "fireworks/deepseek-v3-0324", "fireworks/deepseek-v3p2", "fireworks/deepseek-v3p2-thinking", "fireworks/gpt-oss-120b", "fireworks/gpt-oss-20b", "fireworks/llama4-maverick-instruct-basic", "fireworks/qwen3-235b-a22b", "google/gemini-1.5-flash-002", "google/gemini-1.5-pro-002", "google/gemini-2.0-flash-001", "google/gemini-2.5-flash-lite-preview-09-2025", "google/gemini-2.5-flash-lite-preview-09-2025-thinking", "google/gemini-2.5-flash-preview-09-2025", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-pro-exp-03-25", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "grok/grok-2-1212", "grok/grok-3", "grok/grok-3-mini-fast-high-reasoning", "grok/grok-3-mini-fast-low-reasoning", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "kimi/kimi-k2-thinking", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse_spark", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "minimax/MiniMax-M2.1", "minimax/MiniMax-M2.5", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/magistral-medium-2509", "mistralai/magistral-small-2509", "mistralai/mistral-large-2411", "mistralai/mistral-large-2512", "mistralai/mistral-medium-2505", "mistralai/mistral-medium-3.5", "mistralai/mistral-small-2402", "mistralai/mistral-small-2503", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-4.1-2025-04-14", "openai/gpt-4.1-mini-2025-04-14", "openai/gpt-4.1-nano-2025-04-14", "openai/gpt-4o-2024-08-06", "openai/gpt-4o-2024-11-20", "openai/gpt-4o-mini-2024-07-18", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-nano-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/o1-2024-12-17", "openai/o3-2025-04-16", "openai/o3-mini-2025-01-31", "openai/o4-mini-2025-04-16", "poolside/laguna-m.1", "poolside/laguna-xs.2", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561", "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking", "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "together/meta-llama/Llama-4-Scout-17B-16E-Instruct", "together/moonshotai/Kimi-K2-Instruct", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "zai/glm-4.5", "zai/glm-4.6", "zai/glm-4.7", "zai/glm-5-thinking", "zai/glm-5.1", "zai/glm-5.2", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "platform", "mode": "one-shot", "archived": false, "partner": false, "total_models": 138 }, "tasks": { "overall": { "anthropic/claude-fable-5-1": { "accuracy": 92.375, "latency": 20.589, "stderr": 0.266, "cost_per_test": 0.097724, "avg_input_tokens": 1889.6, "avg_output_tokens": 1483.7, "avg_cache_write_tokens": 1879.8, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 20.6, "median_query_seconds": 8.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 91.591, "latency": 9.6, "stderr": 0.276, "cost_per_test": 0.027843, "avg_input_tokens": 1887.6, "avg_output_tokens": 644.1, "avg_cache_write_tokens": 1876.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 9.6, "median_query_seconds": 6.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 91.502, "latency": 24.998, "stderr": 0.278, "cost_per_test": 0.082276, "avg_input_tokens": 1887.6, "avg_output_tokens": 1219.2, "avg_cache_write_tokens": 1007.4, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 25.0, "median_query_seconds": 11.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 90.987, "latency": 23.912, "stderr": 0.284, "cost_per_test": 0.027545, "avg_input_tokens": 1353.1, "avg_output_tokens": 2069.9, "avg_reasoning_tokens": 1767.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 23.9, "median_query_seconds": 13.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 90.221, "latency": 7.82, "stderr": 0.295, "cost_per_test": 0.022457, "avg_input_tokens": 1352.0, "avg_output_tokens": 2723.9, "avg_reasoning_tokens": 2472.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 7.8, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 90.121, "latency": 4.159, "stderr": 0.296, "cost_per_test": 0.010817, "avg_input_tokens": 1352.0, "avg_output_tokens": 1171.8, "avg_reasoning_tokens": 947.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 4.2, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 90.102, "latency": 24.272, "stderr": 0.295, "cost_per_test": 0.036926, "avg_input_tokens": 1352.9, "avg_output_tokens": 2854.5, "avg_reasoning_tokens": 2467.7, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 24.3, "median_query_seconds": 17.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 89.871, "latency": 187.83, "stderr": 0.305, "cost_per_test": 0.061711, "avg_input_tokens": 1892.6, "avg_output_tokens": 2360.9, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 187.8, "median_query_seconds": 142.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 89.585, "latency": 23.44, "stderr": 0.303, "cost_per_test": 0.060716, "avg_input_tokens": 1887.6, "avg_output_tokens": 2013.4, "avg_cache_write_tokens": 794.7, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 23.4, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 89.515, "latency": 26.262, "stderr": 0.306, "cost_per_test": 0.02498, "avg_input_tokens": 1353.1, "avg_output_tokens": 1856.2, "avg_reasoning_tokens": 1563.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 26.2, "median_query_seconds": 16.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 89.4, "latency": 54.975, "stderr": 0.305, "cost_per_test": 0.015391, "avg_input_tokens": 1324.3, "avg_output_tokens": 2363.9, "avg_reasoning_tokens": 2198.4, "avg_cache_read_tokens": 960.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 55.0, "median_query_seconds": 20.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 89.311, "latency": 29.15, "stderr": 0.305, "cost_per_test": 0.01678, "avg_input_tokens": 1374.6, "avg_output_tokens": 1779.1, "avg_reasoning_tokens": 1446.7, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 29.2, "median_query_seconds": 24.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 89.276, "latency": 7.298, "stderr": 0.305, "cost_per_test": 0.013327, "avg_input_tokens": 1352.0, "avg_output_tokens": 1506.5, "avg_reasoning_tokens": 1252.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 7.3, "median_query_seconds": 5.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 89.216, "latency": 357.751, "stderr": 0.31, "cost_per_test": 0.014974, "avg_input_tokens": 1324.3, "avg_output_tokens": 2317.6, "avg_reasoning_tokens": 2097.9, "avg_cache_read_tokens": 1053.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 357.8, "median_query_seconds": 316.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 89.107, "latency": 49.451, "stderr": 0.455, "cost_per_test": 0.052129, "avg_input_tokens": 1466.3, "avg_output_tokens": 3182.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 49.5, "median_query_seconds": 16.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 89.1, "latency": 15.654, "stderr": 0.308, "cost_per_test": 0.032087, "avg_input_tokens": 1316.3, "avg_output_tokens": 869.0, "avg_reasoning_tokens": 780.7, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 15.6, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 88.726, "latency": 21.03, "stderr": 0.318, "cost_per_test": 0.010704, "avg_input_tokens": 1452.9, "avg_output_tokens": 2387.1, "avg_reasoning_tokens": 2173.4, "avg_cache_read_tokens": 1140.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 21.0, "median_query_seconds": 14.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 88.602, "latency": 63.576, "stderr": 0.313, "cost_per_test": 0.018585, "avg_input_tokens": 1412.6, "avg_output_tokens": 2819.2, "avg_reasoning_tokens": 2646.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 63.6, "median_query_seconds": 20.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 88.592, "latency": 36.249, "stderr": 0.315, "cost_per_test": 0.019364, "avg_input_tokens": 1353.0, "avg_output_tokens": 6232.9, "avg_reasoning_tokens": 5872.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 36.2, "median_query_seconds": 25.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 88.275, "latency": 44.234, "stderr": 0.318, "cost_per_test": 0.010668, "avg_input_tokens": 1295.9, "avg_output_tokens": 2390.7, "avg_reasoning_tokens": 2149.8, "avg_cache_read_tokens": 1009.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 44.2, "median_query_seconds": 24.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.5": { "accuracy": 88.144, "latency": 42.114, "stderr": 0.318, "cost_per_test": 0.027891, "avg_input_tokens": 1322.1, "avg_output_tokens": 1345.0, "avg_reasoning_tokens": 1219.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 42.1, "median_query_seconds": 29.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 87.973, "latency": 38.718, "stderr": 0.321, "cost_per_test": 0.021327, "avg_input_tokens": 1392.5, "avg_output_tokens": 1368.9, "avg_reasoning_tokens": 1176.3, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 38.7, "median_query_seconds": 16.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 87.924, "latency": 27.668, "stderr": 0.325, "cost_per_test": 0.124083, "avg_input_tokens": 1477.4, "avg_output_tokens": 1359.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 27.7, "median_query_seconds": 21.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 87.668, "latency": 46.184, "stderr": 0.323, "cost_per_test": 0.008813, "avg_input_tokens": 1374.5, "avg_output_tokens": 2708.6, "avg_reasoning_tokens": 2453.1, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 46.2, "median_query_seconds": 41.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 87.572, "latency": 156.347, "stderr": 0.33, "cost_per_test": 0.018214, "avg_input_tokens": 1314.4, "avg_output_tokens": 4463.5, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 989.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 156.3, "median_query_seconds": 78.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 87.545, "latency": 25.191, "stderr": 0.368, "cost_per_test": 0.042996, "avg_input_tokens": 1887.6, "avg_output_tokens": 2398.7, "avg_cache_write_tokens": 1868.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 25.2, "median_query_seconds": 7.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 87.482, "latency": 28.875, "stderr": 0.421, "cost_per_test": 0.050888, "avg_input_tokens": 1311.1, "avg_output_tokens": 1657.5, "avg_reasoning_tokens": 1513.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 28.9, "median_query_seconds": 14.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 87.357, "latency": 38.2, "stderr": 0.391, "cost_per_test": 0.040152, "avg_input_tokens": 1477.4, "avg_output_tokens": 2381.3, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 38.2, "median_query_seconds": 27.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 87.341, "latency": 55.406, "stderr": 0.434, "cost_per_test": 0.066412, "avg_input_tokens": 1466.4, "avg_output_tokens": 4124.2, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 55.4, "median_query_seconds": 19.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark": { "accuracy": 87.316, "latency": 46.242, "stderr": 0.327, "cost_per_test": 0.00167, "avg_input_tokens": 1350.9, "avg_output_tokens": 2448.2, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 46.2, "median_query_seconds": 38.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 87.26, "latency": 38.92, "stderr": 0.381, "cost_per_test": 0.215138, "avg_input_tokens": 1478.4, "avg_output_tokens": 2572.8, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 38.9, "median_query_seconds": 23.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 87.249, "latency": 577.617, "stderr": 0.335, "cost_per_test": 0.027032, "avg_input_tokens": 1376.6, "avg_output_tokens": 6662.0, "avg_reasoning_tokens": 6504.9, "avg_cache_read_tokens": 914.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 577.6, "median_query_seconds": 542.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 87.214, "latency": 17.757, "stderr": 0.328, "cost_per_test": 0.007078, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 87.178, "latency": 76.678, "stderr": 0.328, "cost_per_test": 0.008912, "avg_input_tokens": 1374.6, "avg_output_tokens": 3484.2, "avg_reasoning_tokens": 3069.3, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 76.7, "median_query_seconds": 63.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 87.049, "latency": 40.957, "stderr": 0.333, "cost_per_test": 0.003225, "avg_input_tokens": 1312.4, "avg_output_tokens": 2291.3, "avg_cache_read_tokens": 47.0, "avg_cache_write_tokens": 1263.4, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 41.0, "median_query_seconds": 23.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 86.972, "latency": 59.624, "stderr": 0.341, "cost_per_test": 0.01283, "avg_input_tokens": 1389.6, "avg_output_tokens": 3140.7, "avg_reasoning_tokens": 3032.4, "avg_cache_read_tokens": 1129.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 59.6, "median_query_seconds": 16.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.1": { "accuracy": 86.9, "latency": 69.318, "stderr": 0.334, "cost_per_test": 0.012383, "avg_input_tokens": 1317.5, "avg_output_tokens": 3728.7, "avg_reasoning_tokens": 3445.8, "avg_cache_read_tokens": 1083.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 69.3, "median_query_seconds": 32.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3": { "accuracy": 86.77, "latency": 61.783, "stderr": 0.335, "cost_per_test": 0.016639, "avg_input_tokens": 1324.5, "avg_output_tokens": 3511.6, "avg_reasoning_tokens": 3295.3, "avg_cache_read_tokens": 581.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 61.8, "median_query_seconds": 13.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.2": { "accuracy": 86.714, "latency": 51.64, "stderr": 0.334, "cost_per_test": 0.013838, "avg_input_tokens": 1324.5, "avg_output_tokens": 2915.2, "avg_reasoning_tokens": 2630.8, "avg_cache_read_tokens": 736.2, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 51.6, "median_query_seconds": 24.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 86.659, "latency": 8.173, "stderr": 0.333, "cost_per_test": 0.008039, "avg_input_tokens": 1309.7, "avg_output_tokens": 472.0, "avg_reasoning_tokens": 386.9, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 8.2, "median_query_seconds": 4.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 86.544, "latency": 37.239, "stderr": 0.336, "cost_per_test": 0.024135, "avg_input_tokens": 1308.9, "avg_output_tokens": 2250.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 37.2, "median_query_seconds": 24.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 86.377, "latency": 23.056, "stderr": 0.337, "cost_per_test": 0.018514, "avg_input_tokens": 1309.3, "avg_output_tokens": 1724.1, "avg_reasoning_tokens": 1524.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 23.1, "median_query_seconds": 14.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 86.295, "latency": 73.264, "stderr": 0.34, "cost_per_test": 0.039629, "avg_input_tokens": 1317.0, "avg_output_tokens": 4066.8, "avg_cache_read_tokens": 1123.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 73.2, "median_query_seconds": 23.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 86.254, "latency": 11.654, "stderr": 0.34, "cost_per_test": 0.012836, "avg_input_tokens": 1395.7, "avg_output_tokens": 2030.5, "avg_reasoning_tokens": 1824.6, "avg_cache_read_tokens": 1187.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 11.7, "median_query_seconds": 6.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 86.24, "latency": 14.511, "stderr": 0.34, "cost_per_test": 0.001052, "avg_input_tokens": 1352.0, "avg_output_tokens": 2292.5, "avg_reasoning_tokens": 1968.3, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 14.5, "median_query_seconds": 6.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 86.234, "latency": 35.413, "stderr": 0.344, "cost_per_test": 0.03698, "avg_input_tokens": 1317.3, "avg_output_tokens": 4083.3, "avg_reasoning_tokens": 2375.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 35.4, "median_query_seconds": 15.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 86.206, "latency": 17.188, "stderr": 0.339, "cost_per_test": 0.00265, "avg_input_tokens": 1376.6, "avg_output_tokens": 1912.9, "avg_reasoning_tokens": 1822.6, "avg_cache_read_tokens": 1129.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 17.2, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 86.166, "latency": 10.405, "stderr": 0.337, "cost_per_test": 0.049652, "avg_input_tokens": 1449.4, "avg_output_tokens": 372.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 10.4, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 86.059, "latency": 44.762, "stderr": 0.341, "cost_per_test": 0.001058, "avg_input_tokens": 1324.5, "avg_output_tokens": 4019.0, "avg_cache_read_tokens": 777.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 44.8, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 86.036, "latency": 16.948, "stderr": 0.35, "cost_per_test": 0.002842, "avg_input_tokens": 1314.4, "avg_output_tokens": 2151.7, "avg_reasoning_tokens": 2053.1, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 17.0, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 86.03, "latency": 86.889, "stderr": 0.343, "cost_per_test": 0.011542, "avg_input_tokens": 1317.5, "avg_output_tokens": 3462.3, "avg_cache_read_tokens": 1036.2, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 86.9, "median_query_seconds": 60.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 85.914, "latency": 39.204, "stderr": 0.343, "cost_per_test": 0.00793, "avg_input_tokens": 1314.4, "avg_output_tokens": 2550.2, "avg_cache_read_tokens": 1002.2, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 39.2, "median_query_seconds": 27.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.3": { "accuracy": 85.838, "latency": 637.088, "stderr": 0.33, "cost_per_test": 0.009489, "avg_input_tokens": 1281.5, "avg_output_tokens": 3587.7, "avg_reasoning_tokens": 3328.5, "avg_cache_read_tokens": 1030.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 637.1, "median_query_seconds": 603.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 85.836, "latency": 4.823, "stderr": 0.344, "cost_per_test": 0.004522, "avg_input_tokens": 1352.0, "avg_output_tokens": 1646.6, "avg_reasoning_tokens": 1391.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 4.8, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 85.761, "latency": 32.92, "stderr": 0.345, "cost_per_test": null, "avg_input_tokens": 1361.6, "avg_output_tokens": 3118.9, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 32.9, "median_query_seconds": 12.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 85.595, "latency": 16.735, "stderr": 0.345, "cost_per_test": 0.011827, "avg_input_tokens": 1309.0, "avg_output_tokens": 1151.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 16.7, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 85.59, "latency": 8.248, "stderr": 0.367, "cost_per_test": 0.052065, "avg_input_tokens": 1449.4, "avg_output_tokens": 404.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 8.2, "median_query_seconds": 8.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 85.566, "latency": 71.685, "stderr": 0.346, "cost_per_test": 0.005706, "avg_input_tokens": 1317.0, "avg_output_tokens": 4655.9, "avg_cache_read_tokens": 1150.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 71.7, "median_query_seconds": 23.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4-0709": { "accuracy": 85.304, "latency": 89.255, "stderr": 0.349, "cost_per_test": 0.05025, "avg_input_tokens": 1275.6, "avg_output_tokens": 3094.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 89.3, "median_query_seconds": 47.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 84.976, "latency": 110.207, "stderr": 0.352, "cost_per_test": 0.026557, "avg_input_tokens": 1357.4, "avg_output_tokens": 4154.7, "avg_reasoning_tokens": 3547.7, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 110.2, "median_query_seconds": 78.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 84.917, "latency": 90.148, "stderr": 0.353, "cost_per_test": 0.001102, "avg_input_tokens": 1297.6, "avg_output_tokens": 2490.8, "avg_reasoning_tokens": 2382.4, "avg_cache_read_tokens": 1188.1, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 90.2, "median_query_seconds": 57.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 84.595, "latency": 67.757, "stderr": 0.453, "cost_per_test": 0.002085, "avg_input_tokens": 1598.4, "avg_output_tokens": 2193.6, "avg_reasoning_tokens": 1981.7, "avg_cache_read_tokens": 1196.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 67.8, "median_query_seconds": 36.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 84.554, "latency": 20.001, "stderr": 0.359, "cost_per_test": 0.00368, "avg_input_tokens": 1309.0, "avg_output_tokens": 1695.2, "avg_reasoning_tokens": 1562.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 20.0, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 84.36, "latency": 0.0, "stderr": 0.356, "cost_per_test": 0.006426, "avg_input_tokens": 1355.5, "avg_output_tokens": 799.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 84.337, "latency": 69.848, "stderr": 0.356, "cost_per_test": 0.010543, "avg_input_tokens": 1416.5, "avg_output_tokens": 3278.4, "avg_reasoning_tokens": 3105.3, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 69.8, "median_query_seconds": 26.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 84.22, "latency": 41.39, "stderr": 0.364, "cost_per_test": 0.004811, "avg_input_tokens": 1450.4, "avg_output_tokens": 1878.7, "avg_cache_read_tokens": 1183.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 41.4, "median_query_seconds": 19.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 84.18, "latency": 25.469, "stderr": 0.357, "cost_per_test": 0.001078, "avg_input_tokens": 1424.6, "avg_output_tokens": 1931.6, "avg_reasoning_tokens": 1790.5, "avg_cache_read_tokens": 1154.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 25.5, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 84.061, "latency": 33.303, "stderr": 0.36, "cost_per_test": 0.001734, "avg_input_tokens": 1374.6, "avg_output_tokens": 3991.8, "avg_reasoning_tokens": 3546.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 33.3, "median_query_seconds": 28.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 84.058, "latency": 18.46, "stderr": 0.358, "cost_per_test": 0.009006, "avg_input_tokens": 1351.6, "avg_output_tokens": 731.6, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 18.5, "median_query_seconds": 15.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 83.865, "latency": 47.087, "stderr": 0.361, "cost_per_test": 0.045537, "avg_input_tokens": 1477.4, "avg_output_tokens": 2740.3, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 47.1, "median_query_seconds": 33.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 83.686, "latency": 8.525, "stderr": 0.362, "cost_per_test": 0.001454, "avg_input_tokens": 1351.6, "avg_output_tokens": 1729.1, "avg_reasoning_tokens": 1309.9, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 8.5, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 83.664, "latency": 10.445, "stderr": 0.361, "cost_per_test": 0.00146, "avg_input_tokens": 1351.6, "avg_output_tokens": 1736.4, "avg_reasoning_tokens": 1314.8, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 10.4, "median_query_seconds": 9.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 83.537, "latency": 43.428, "stderr": 0.364, "cost_per_test": 0.006194, "avg_input_tokens": 1355.4, "avg_output_tokens": 761.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 43.4, "median_query_seconds": 28.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 83.488, "latency": 26.867, "stderr": 0.364, "cost_per_test": 0.116223, "avg_input_tokens": 1309.0, "avg_output_tokens": 1609.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 26.9, "median_query_seconds": 19.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 83.184, "latency": 27.278, "stderr": 0.444, "cost_per_test": 0.012339, "avg_input_tokens": 1296.6, "avg_output_tokens": 1960.8, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 27.3, "median_query_seconds": 17.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 83.056, "latency": 40.772, "stderr": 0.381, "cost_per_test": 0.000283, "avg_input_tokens": 1297.5, "avg_output_tokens": 489.5, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1131.6, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 40.8, "median_query_seconds": 31.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 82.931, "latency": 30.248, "stderr": 0.372, "cost_per_test": 0.000774, "avg_input_tokens": 1594.4, "avg_output_tokens": 2528.7, "avg_reasoning_tokens": 2370.4, "avg_cache_read_tokens": 1142.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 30.2, "median_query_seconds": 18.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-4.7": { "accuracy": 82.736, "latency": 104.451, "stderr": 0.371, "cost_per_test": 0.00798, "avg_input_tokens": 1317.4, "avg_output_tokens": 3486.7, "avg_cache_read_tokens": 898.1, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 104.4, "median_query_seconds": 53.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 82.728, "latency": 31.744, "stderr": 0.37, "cost_per_test": 0.040182, "avg_input_tokens": 1477.4, "avg_output_tokens": 2383.3, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 31.8, "median_query_seconds": 24.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 82.226, "latency": 22.196, "stderr": 0.373, "cost_per_test": 0.005196, "avg_input_tokens": 1309.0, "avg_output_tokens": 2435.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 22.2, "median_query_seconds": 15.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.6": { "accuracy": 82.203, "latency": 46.996, "stderr": 0.375, "cost_per_test": 0.006572, "avg_input_tokens": 1318.9, "avg_output_tokens": 2631.3, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 47.0, "median_query_seconds": 28.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 82.008, "latency": 20.073, "stderr": 0.375, "cost_per_test": 0.000862, "avg_input_tokens": 1376.8, "avg_output_tokens": 3734.6, "avg_cache_read_tokens": 1056.2, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 20.1, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 81.374, "latency": 16.888, "stderr": 0.382, "cost_per_test": 0.001758, "avg_input_tokens": 1275.6, "avg_output_tokens": 2750.6, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 16.9, "median_query_seconds": 11.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 81.246, "latency": 52.354, "stderr": 0.387, "cost_per_test": 0.002316, "avg_input_tokens": 1355.4, "avg_output_tokens": 2293.4, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 52.4, "median_query_seconds": 30.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-4.5": { "accuracy": 81.222, "latency": 136.72, "stderr": 0.378, "cost_per_test": 0.009757, "avg_input_tokens": 1317.6, "avg_output_tokens": 4075.6, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 136.7, "median_query_seconds": 80.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 81.068, "latency": 94.771, "stderr": 0.396, "cost_per_test": 0.002067, "avg_input_tokens": 1314.2, "avg_output_tokens": 2864.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 94.8, "median_query_seconds": 52.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 80.663, "latency": 6.302, "stderr": 0.383, "cost_per_test": 0.009664, "avg_input_tokens": 1449.4, "avg_output_tokens": 354.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 6.3, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 80.561, "latency": 10.443, "stderr": 0.386, "cost_per_test": 0.005524, "avg_input_tokens": 1309.0, "avg_output_tokens": 928.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 10.4, "median_query_seconds": 8.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 80.495, "latency": 8.177, "stderr": 0.385, "cost_per_test": 0.007141, "avg_input_tokens": 1310.0, "avg_output_tokens": 565.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 8.2, "median_query_seconds": 3.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 80.427, "latency": 50.005, "stderr": 0.39, "cost_per_test": 0.003947, "avg_input_tokens": 1295.8, "avg_output_tokens": 2893.7, "avg_cache_read_tokens": 33.3, "avg_cache_write_tokens": 1262.6, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 50.0, "median_query_seconds": 22.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 80.093, "latency": 48.271, "stderr": 0.39, "cost_per_test": 0.005362, "avg_input_tokens": 1315.4, "avg_output_tokens": 2035.9, "avg_cache_read_tokens": 49.4, "avg_cache_write_tokens": 1266.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 48.3, "median_query_seconds": 29.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 80.011, "latency": 7.426, "stderr": 0.39, "cost_per_test": 0.001026, "avg_input_tokens": 1275.6, "avg_output_tokens": 1287.3, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 7.4, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-3": { "accuracy": 79.949, "latency": 12.096, "stderr": 0.39, "cost_per_test": 0.016622, "avg_input_tokens": 1274.6, "avg_output_tokens": 853.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 12.1, "median_query_seconds": 10.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 79.823, "latency": 13.554, "stderr": 0.472, "cost_per_test": 0.001481, "avg_input_tokens": 1348.6, "avg_output_tokens": 539.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 13.6, "median_query_seconds": 11.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 79.704, "latency": 7.343, "stderr": 0.393, "cost_per_test": 0.001742, "avg_input_tokens": 1385.6, "avg_output_tokens": 2930.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 7.3, "median_query_seconds": 4.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 79.474, "latency": 25.051, "stderr": 0.396, "cost_per_test": 0.00147, "avg_input_tokens": 1296.6, "avg_output_tokens": 337.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 25.0, "median_query_seconds": 20.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 79.432, "latency": 9.707, "stderr": 0.372, "cost_per_test": 0.010229, "avg_input_tokens": 1449.0, "avg_output_tokens": 392.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 9.7, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 79.415, "latency": 6.803, "stderr": 0.395, "cost_per_test": 0.000686, "avg_input_tokens": 1298.9, "avg_output_tokens": 454.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 6.8, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 79.394, "latency": 20.338, "stderr": 0.394, "cost_per_test": 0.002836, "avg_input_tokens": 1323.5, "avg_output_tokens": 504.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 20.3, "median_query_seconds": 15.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 79.166, "latency": 35.41, "stderr": 0.395, "cost_per_test": 0.001438, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 79.118, "latency": 9.618, "stderr": 0.397, "cost_per_test": 0.000314, "avg_input_tokens": 1351.6, "avg_output_tokens": 2187.5, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 9.6, "median_query_seconds": 8.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 78.715, "latency": 25.784, "stderr": 0.477, "cost_per_test": 0.018312, "avg_input_tokens": 1477.4, "avg_output_tokens": 3367.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 25.8, "median_query_seconds": 16.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 78.689, "latency": 22.375, "stderr": 0.394, "cost_per_test": 0.012991, "avg_input_tokens": 1309.0, "avg_output_tokens": 2625.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 22.4, "median_query_seconds": 15.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 78.639, "latency": 3.19, "stderr": 0.4, "cost_per_test": 0.000556, "avg_input_tokens": 1351.6, "avg_output_tokens": 1051.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 3.2, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 78.404, "latency": 6.288, "stderr": 0.397, "cost_per_test": 0.00859, "avg_input_tokens": 1449.4, "avg_output_tokens": 282.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 6.3, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 77.376, "latency": 4.318, "stderr": 0.407, "cost_per_test": 0.000414, "avg_input_tokens": 1351.6, "avg_output_tokens": 696.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 4.3, "median_query_seconds": 2.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 77.225, "latency": 3.832, "stderr": 0.405, "cost_per_test": 0.001316, "avg_input_tokens": 1310.0, "avg_output_tokens": 495.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 3.8, "median_query_seconds": 3.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 77.172, "latency": 8.21, "stderr": 0.433, "cost_per_test": 0.000396, "avg_input_tokens": 1310.9, "avg_output_tokens": 828.0, "avg_reasoning_tokens": 661.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 8.2, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 76.067, "latency": 26.107, "stderr": 0.412, "cost_per_test": 0.001887, "avg_input_tokens": 1309.3, "avg_output_tokens": 4566.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 26.1, "median_query_seconds": 20.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-2-1212": { "accuracy": 75.473, "latency": 8.747, "stderr": 0.418, "cost_per_test": 0.005966, "avg_input_tokens": 1328.9, "avg_output_tokens": 330.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 8.7, "median_query_seconds": 5.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 75.335, "latency": 66.384, "stderr": 0.418, "cost_per_test": 0.041763, "avg_input_tokens": 1360.9, "avg_output_tokens": 5296.2, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 66.4, "median_query_seconds": 60.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 75.294, "latency": 3.343, "stderr": 0.417, "cost_per_test": 0.002649, "avg_input_tokens": 1351.6, "avg_output_tokens": 191.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 3.4, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 75.29, "latency": 8.831, "stderr": 0.421, "cost_per_test": 0.000209, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 75.214, "latency": 3.098, "stderr": 0.419, "cost_per_test": 0.000217, "avg_input_tokens": 1436.6, "avg_output_tokens": 205.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1151.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 3.1, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 74.13, "latency": 8.998, "stderr": 0.42, "cost_per_test": 0.007217, "avg_input_tokens": 1310.0, "avg_output_tokens": 394.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 9.0, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 73.82, "latency": 11.36, "stderr": 0.424, "cost_per_test": 0.001338, "avg_input_tokens": 1296.6, "avg_output_tokens": 190.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 11.4, "median_query_seconds": 8.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 72.563, "latency": 9.143, "stderr": 0.441, "cost_per_test": 0.007465, "avg_input_tokens": 1310.0, "avg_output_tokens": 419.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 9.2, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 71.636, "latency": 30.207, "stderr": 0.447, "cost_per_test": 0.00062, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 70.778, "latency": 17.45, "stderr": 0.45, "cost_per_test": null, "avg_input_tokens": 1338.9, "avg_output_tokens": 804.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 17.4, "median_query_seconds": 16.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 70.342, "latency": 2.135, "stderr": 0.431, "cost_per_test": 0.001084, "avg_input_tokens": 1395.6, "avg_output_tokens": 1609.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.1, "avg_query_seconds": 2.1, "median_query_seconds": 1.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 69.86, "latency": 4.179, "stderr": 0.442, "cost_per_test": 0.001565, "avg_input_tokens": 1345.9, "avg_output_tokens": 432.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 4.2, "median_query_seconds": 3.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 69.706, "latency": 7.192, "stderr": 0.442, "cost_per_test": 0.004997, "avg_input_tokens": 1476.7, "avg_output_tokens": 340.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 7.2, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 69.632, "latency": 5.342, "stderr": 0.445, "cost_per_test": 0.000507, "avg_input_tokens": 1298.9, "avg_output_tokens": 463.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 5.3, "median_query_seconds": 5.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 69.577, "latency": 36.351, "stderr": 0.441, "cost_per_test": null, "avg_input_tokens": 1339.5, "avg_output_tokens": 1686.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 36.3, "median_query_seconds": 28.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 69.172, "latency": 9.853, "stderr": 0.463, "cost_per_test": 0.007902, "avg_input_tokens": 1365.0, "avg_output_tokens": 449.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 9.9, "median_query_seconds": 7.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 69.05, "latency": 47.986, "stderr": 0.48, "cost_per_test": 0.000605, "avg_input_tokens": 1419.8, "avg_output_tokens": 2728.2, "avg_reasoning_tokens": 2334.7, "avg_cache_read_tokens": 1197.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 48.0, "median_query_seconds": 32.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-m.1": { "accuracy": 68.84, "latency": 99.583, "stderr": 0.452, "cost_per_test": 0.001794, "avg_input_tokens": 1382.8, "avg_output_tokens": 6351.3, "avg_reasoning_tokens": 3812.1, "avg_cache_read_tokens": 1187.8, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 99.6, "median_query_seconds": 67.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 68.659, "latency": 31.988, "stderr": 0.433, "cost_per_test": 0.014664, "avg_input_tokens": 1348.6, "avg_output_tokens": 2394.4, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 32.0, "median_query_seconds": 20.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 66.021, "latency": 3.602, "stderr": 0.455, "cost_per_test": 0.000203, "avg_input_tokens": 1348.6, "avg_output_tokens": 340.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 3.6, "median_query_seconds": 2.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 65.606, "latency": 1.682, "stderr": 0.456, "cost_per_test": 0.00017, "avg_input_tokens": 1351.6, "avg_output_tokens": 227.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 1.7, "median_query_seconds": 1.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 64.443, "latency": 4.665, "stderr": 0.461, "cost_per_test": 0.000502, "avg_input_tokens": 1348.6, "avg_output_tokens": 386.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 64.117, "latency": 5.792, "stderr": 0.46, "cost_per_test": 0.002224, "avg_input_tokens": 1449.4, "avg_output_tokens": 266.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 5.8, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 63.479, "latency": 2.403, "stderr": 0.462, "cost_per_test": 0.000337, "avg_input_tokens": 1310.0, "avg_output_tokens": 515.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 2.4, "median_query_seconds": 2.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 62.735, "latency": 5.086, "stderr": 0.461, "cost_per_test": 0.000412, "avg_input_tokens": 1310.0, "avg_output_tokens": 360.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 5.1, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 62.135, "latency": 16.124, "stderr": 0.452, "cost_per_test": 0.00355, "avg_input_tokens": 1348.6, "avg_output_tokens": 1918.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 16.1, "median_query_seconds": 12.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 49.778, "latency": 9.48, "stderr": 0.475, "cost_per_test": 0.004628, "avg_input_tokens": 1417.2, "avg_output_tokens": 224.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 9.5, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "cohere/command-r-plus": { "accuracy": 44.003, "latency": 4.419, "stderr": 0.479, "cost_per_test": 0.00566, "avg_input_tokens": 1365.0, "avg_output_tokens": 224.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 4.4, "median_query_seconds": 3.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 30.278, "latency": 2.426, "stderr": 0.448, "cost_per_test": 0.000391, "avg_input_tokens": 1417.2, "avg_output_tokens": 269.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 2.4, "median_query_seconds": 1.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "biology": { "anthropic/claude-fable-5-1": { "accuracy": 96.095, "latency": 11.203, "stderr": 0.723, "cost_per_test": 0.058467, "avg_input_tokens": 2013.9, "avg_output_tokens": 666.1, "avg_cache_write_tokens": 2009.8, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 11.2, "median_query_seconds": 8.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 95.676, "latency": 7.056, "stderr": 0.76, "cost_per_test": 0.022565, "avg_input_tokens": 2011.8, "avg_output_tokens": 399.8, "avg_reasoning_tokens": 139.2, "avg_cache_write_tokens": 2009.8, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 7.1, "median_query_seconds": 6.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 95.397, "latency": 17.19, "stderr": 0.783, "cost_per_test": 0.026082, "avg_input_tokens": 1371.5, "avg_output_tokens": 1944.9, "avg_reasoning_tokens": 1643.1, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 17.2, "median_query_seconds": 12.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 95.258, "latency": 15.298, "stderr": 0.794, "cost_per_test": 0.016695, "avg_input_tokens": 1371.5, "avg_output_tokens": 1162.7, "avg_reasoning_tokens": 921.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 15.3, "median_query_seconds": 9.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 95.119, "latency": 21.48, "stderr": 0.805, "cost_per_test": 0.012963, "avg_input_tokens": 1436.3, "avg_output_tokens": 1249.6, "avg_reasoning_tokens": 977.7, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 21.5, "median_query_seconds": 18.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 94.979, "latency": 5.07, "stderr": 0.816, "cost_per_test": 0.013544, "avg_input_tokens": 1371.5, "avg_output_tokens": 1531.6, "avg_reasoning_tokens": 1349.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 5.1, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 94.979, "latency": 15.731, "stderr": 0.896, "cost_per_test": 0.008217, "avg_input_tokens": 1482.2, "avg_output_tokens": 1796.2, "avg_reasoning_tokens": 1572.4, "avg_cache_read_tokens": 1154.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 15.7, "median_query_seconds": 12.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 94.979, "latency": 26.674, "stderr": 0.816, "cost_per_test": 0.009049, "avg_input_tokens": 1337.2, "avg_output_tokens": 1324.1, "avg_reasoning_tokens": 1180.1, "avg_cache_read_tokens": 1046.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 26.7, "median_query_seconds": 12.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 94.7, "latency": 3.04, "stderr": 0.837, "cost_per_test": 0.007503, "avg_input_tokens": 1371.5, "avg_output_tokens": 726.1, "avg_reasoning_tokens": 562.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 3.0, "median_query_seconds": 2.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 94.7, "latency": 4.901, "stderr": 0.837, "cost_per_test": 0.009068, "avg_input_tokens": 1371.5, "avg_output_tokens": 934.8, "avg_reasoning_tokens": 734.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 4.9, "median_query_seconds": 3.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 94.7, "latency": 16.602, "stderr": 0.837, "cost_per_test": 0.042, "avg_input_tokens": 2011.8, "avg_output_tokens": 1240.4, "avg_cache_read_tokens": 2.8, "avg_cache_write_tokens": 755.5, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 16.6, "median_query_seconds": 9.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 94.7, "latency": 25.614, "stderr": 0.837, "cost_per_test": 0.013204, "avg_input_tokens": 1371.5, "avg_output_tokens": 4172.8, "avg_reasoning_tokens": 3895.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 25.6, "median_query_seconds": 13.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 94.561, "latency": 7.724, "stderr": 0.847, "cost_per_test": 0.018769, "avg_input_tokens": 1362.4, "avg_output_tokens": 398.5, "avg_reasoning_tokens": 325.2, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 7.7, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 94.561, "latency": 64.371, "stderr": 0.847, "cost_per_test": 0.03085, "avg_input_tokens": 2016.7, "avg_output_tokens": 1291.7, "avg_cache_write_tokens": 760.1, "turn_metrics": { "tasks_measured": 713, "avg_turns": 1.0, "avg_query_seconds": 64.4, "median_query_seconds": 51.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 94.282, "latency": 34.732, "stderr": 0.867, "cost_per_test": 0.017451, "avg_input_tokens": 1370.4, "avg_output_tokens": 1700.5, "avg_cache_read_tokens": 1200.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 34.7, "median_query_seconds": 11.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5-thinking": { "accuracy": 94.282, "latency": 57.433, "stderr": 0.867, "cost_per_test": 0.007637, "avg_input_tokens": 1376.6, "avg_output_tokens": 2234.2, "avg_cache_read_tokens": 1084.9, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 57.4, "median_query_seconds": 43.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.5": { "accuracy": 94.282, "latency": 377.984, "stderr": 0.867, "cost_per_test": 0.009436, "avg_input_tokens": 1337.2, "avg_output_tokens": 1398.9, "avg_reasoning_tokens": 1210.5, "avg_cache_read_tokens": 1088.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 378.0, "median_query_seconds": 252.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 94.142, "latency": 8.357, "stderr": 0.877, "cost_per_test": 0.009299, "avg_input_tokens": 1419.7, "avg_output_tokens": 1421.2, "avg_reasoning_tokens": 1199.4, "avg_cache_read_tokens": 1148.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 8.4, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 94.142, "latency": 26.173, "stderr": 0.877, "cost_per_test": 0.007722, "avg_input_tokens": 1474.3, "avg_output_tokens": 1093.9, "avg_reasoning_tokens": 963.2, "avg_cache_read_tokens": 1022.6, "turn_metrics": { "tasks_measured": 716, "avg_turns": 1.0, "avg_query_seconds": 26.2, "median_query_seconds": 10.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 94.142, "latency": 55.706, "stderr": 0.877, "cost_per_test": 0.006883, "avg_input_tokens": 1436.3, "avg_output_tokens": 2628.5, "avg_reasoning_tokens": 2273.8, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 55.7, "median_query_seconds": 43.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 94.003, "latency": 8.647, "stderr": 0.994, "cost_per_test": 0.05251, "avg_input_tokens": 1482.4, "avg_output_tokens": 403.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 8.6, "median_query_seconds": 8.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 94.003, "latency": 20.744, "stderr": 1.843, "cost_per_test": 0.022805, "avg_input_tokens": 1499.4, "avg_output_tokens": 1220.5, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 20.7, "median_query_seconds": 14.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 94.003, "latency": 24.475, "stderr": 0.887, "cost_per_test": 0.018287, "avg_input_tokens": 1362.4, "avg_output_tokens": 788.9, "avg_reasoning_tokens": 678.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 24.5, "median_query_seconds": 19.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 94.003, "latency": 34.071, "stderr": 0.887, "cost_per_test": 0.007903, "avg_input_tokens": 1383.8, "avg_output_tokens": 1569.3, "avg_reasoning_tokens": 1355.3, "avg_cache_read_tokens": 824.2, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 34.1, "median_query_seconds": 19.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 93.863, "latency": 12.325, "stderr": 0.896, "cost_per_test": 0.015872, "avg_input_tokens": 1371.5, "avg_output_tokens": 1094.1, "avg_reasoning_tokens": 869.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 12.3, "median_query_seconds": 7.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 93.863, "latency": 15.082, "stderr": 0.896, "cost_per_test": 0.0126, "avg_input_tokens": 1447.6, "avg_output_tokens": 785.0, "avg_reasoning_tokens": 555.4, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 15.1, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 93.863, "latency": 19.649, "stderr": 0.915, "cost_per_test": 0.000484, "avg_input_tokens": 1383.8, "avg_output_tokens": 1660.8, "avg_cache_read_tokens": 586.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 19.6, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 93.863, "latency": 28.191, "stderr": 0.924, "cost_per_test": 0.006943, "avg_input_tokens": 1325.4, "avg_output_tokens": 1516.2, "avg_reasoning_tokens": 1278.7, "avg_cache_read_tokens": 1052.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 715, "avg_turns": 1.0, "avg_query_seconds": 28.2, "median_query_seconds": 17.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 93.724, "latency": 24.916, "stderr": 1.122, "cost_per_test": 0.139523, "avg_input_tokens": 1511.4, "avg_output_tokens": 1558.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 24.9, "median_query_seconds": 18.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 93.724, "latency": 28.033, "stderr": 0.924, "cost_per_test": 0.00547, "avg_input_tokens": 1413.1, "avg_output_tokens": 1271.7, "avg_reasoning_tokens": 1185.1, "avg_cache_read_tokens": 1121.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 28.0, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.1": { "accuracy": 93.724, "latency": 29.038, "stderr": 0.906, "cost_per_test": 0.005623, "avg_input_tokens": 1376.8, "avg_output_tokens": 1609.1, "avg_reasoning_tokens": 1324.6, "avg_cache_read_tokens": 1129.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 29.0, "median_query_seconds": 16.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark": { "accuracy": 93.724, "latency": 34.668, "stderr": 0.906, "cost_per_test": 0.001406, "avg_input_tokens": 1380.2, "avg_output_tokens": 1997.8, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 34.7, "median_query_seconds": 32.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 93.724, "latency": 35.767, "stderr": 0.906, "cost_per_test": 0.007006, "avg_input_tokens": 1436.1, "avg_output_tokens": 2096.1, "avg_reasoning_tokens": 1881.1, "turn_metrics": { "tasks_measured": 716, "avg_turns": 1.0, "avg_query_seconds": 35.8, "median_query_seconds": 32.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 93.724, "latency": 96.93, "stderr": 0.906, "cost_per_test": 0.010782, "avg_input_tokens": 1369.6, "avg_output_tokens": 2570.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1014.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 716, "avg_turns": 1.0, "avg_query_seconds": 96.9, "median_query_seconds": 54.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 93.584, "latency": 18.862, "stderr": 0.915, "cost_per_test": 0.086148, "avg_input_tokens": 2011.8, "avg_output_tokens": 1227.3, "avg_cache_write_tokens": 1961.6, "turn_metrics": { "tasks_measured": 715, "avg_turns": 1.0, "avg_query_seconds": 18.9, "median_query_seconds": 12.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 93.584, "latency": 27.098, "stderr": 0.924, "cost_per_test": 0.0022, "avg_input_tokens": 1335.3, "avg_output_tokens": 1428.3, "avg_cache_read_tokens": 41.8, "avg_cache_write_tokens": 1291.5, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 27.1, "median_query_seconds": 18.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 93.584, "latency": 33.745, "stderr": 1.443, "cost_per_test": 0.05724, "avg_input_tokens": 2011.8, "avg_output_tokens": 3313.2, "avg_cache_write_tokens": 2009.8, "turn_metrics": { "tasks_measured": 714, "avg_turns": 1.0, "avg_query_seconds": 33.7, "median_query_seconds": 12.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 93.305, "latency": 8.764, "stderr": 0.96, "cost_per_test": 0.001241, "avg_input_tokens": 1362.4, "avg_output_tokens": 808.4, "avg_reasoning_tokens": 728.6, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 8.8, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 93.305, "latency": 32.613, "stderr": 0.933, "cost_per_test": 0.007849, "avg_input_tokens": 1383.8, "avg_output_tokens": 1496.3, "avg_reasoning_tokens": 1356.7, "avg_cache_read_tokens": 589.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 32.6, "median_query_seconds": 11.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 93.305, "latency": 40.714, "stderr": 0.933, "cost_per_test": 0.002762, "avg_input_tokens": 1370.5, "avg_output_tokens": 2202.4, "avg_cache_read_tokens": 1217.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 715, "avg_turns": 1.0, "avg_query_seconds": 40.7, "median_query_seconds": 11.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.3": { "accuracy": 93.305, "latency": 314.075, "stderr": 0.933, "cost_per_test": 0.00588, "avg_input_tokens": 1302.0, "avg_output_tokens": 2149.7, "avg_reasoning_tokens": 1909.2, "avg_cache_read_tokens": 1068.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 314.1, "median_query_seconds": 322.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 93.166, "latency": 21.666, "stderr": 0.942, "cost_per_test": 0.000816, "avg_input_tokens": 1371.5, "avg_output_tokens": 1697.2, "avg_reasoning_tokens": 1412.7, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 21.7, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 93.026, "latency": 9.097, "stderr": 0.951, "cost_per_test": 0.00555, "avg_input_tokens": 1362.4, "avg_output_tokens": 235.4, "avg_reasoning_tokens": 168.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 9.1, "median_query_seconds": 3.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 93.026, "latency": 10.237, "stderr": 0.951, "cost_per_test": 0.048629, "avg_input_tokens": 1482.4, "avg_output_tokens": 351.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 10.2, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 93.026, "latency": 13.658, "stderr": 0.951, "cost_per_test": 0.00067, "avg_input_tokens": 1445.0, "avg_output_tokens": 1098.0, "avg_reasoning_tokens": 948.4, "avg_cache_read_tokens": 1120.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 13.7, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 93.026, "latency": 24.712, "stderr": 0.951, "cost_per_test": 0.01654, "avg_input_tokens": 1362.4, "avg_output_tokens": 1483.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 24.7, "median_query_seconds": 19.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 93.026, "latency": 24.822, "stderr": 0.951, "cost_per_test": 0.00137, "avg_input_tokens": 1436.3, "avg_output_tokens": 3066.5, "avg_reasoning_tokens": 2684.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 24.8, "median_query_seconds": 20.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 93.026, "latency": 28.765, "stderr": 0.96, "cost_per_test": 0.005674, "avg_input_tokens": 1369.5, "avg_output_tokens": 1786.0, "avg_cache_read_tokens": 995.4, "turn_metrics": { "tasks_measured": 716, "avg_turns": 1.0, "avg_query_seconds": 28.8, "median_query_seconds": 20.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 92.887, "latency": 10.554, "stderr": 0.96, "cost_per_test": 0.001333, "avg_input_tokens": 1400.1, "avg_output_tokens": 911.7, "avg_reasoning_tokens": 841.6, "avg_cache_read_tokens": 1141.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 10.6, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 92.887, "latency": 14.15, "stderr": 1.53, "cost_per_test": 0.027279, "avg_input_tokens": 1362.4, "avg_output_tokens": 847.5, "avg_reasoning_tokens": 736.4, "avg_cache_read_tokens": 459.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 14.2, "median_query_seconds": 9.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 92.887, "latency": 19.899, "stderr": 0.96, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 92.887, "latency": 636.88, "stderr": 0.96, "cost_per_test": 0.012403, "avg_input_tokens": 1400.1, "avg_output_tokens": 3024.1, "avg_reasoning_tokens": 2897.9, "avg_cache_read_tokens": 1113.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 636.9, "median_query_seconds": 612.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3-max": { "accuracy": 92.748, "latency": 0.0, "stderr": 0.969, "cost_per_test": 0.005556, "avg_input_tokens": 1419.5, "avg_output_tokens": 642.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 92.748, "latency": 25.286, "stderr": 1.305, "cost_per_test": 0.030244, "avg_input_tokens": 1499.4, "avg_output_tokens": 1715.6, "avg_cache_write_tokens": 15.3, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 25.3, "median_query_seconds": 15.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 92.748, "latency": 27.899, "stderr": 1.115, "cost_per_test": 0.028668, "avg_input_tokens": 1510.4, "avg_output_tokens": 1609.1, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 27.9, "median_query_seconds": 20.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 92.748, "latency": 33.196, "stderr": 1.01, "cost_per_test": 0.002851, "avg_input_tokens": 1473.3, "avg_output_tokens": 1054.6, "avg_cache_read_tokens": 1174.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 33.2, "median_query_seconds": 23.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 92.608, "latency": 10.958, "stderr": 0.977, "cost_per_test": 0.008551, "avg_input_tokens": 1362.4, "avg_output_tokens": 728.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 11.0, "median_query_seconds": 8.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 92.608, "latency": 14.363, "stderr": 0.977, "cost_per_test": 0.0082, "avg_input_tokens": 1381.7, "avg_output_tokens": 647.3, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 14.4, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 92.608, "latency": 16.66, "stderr": 1.018, "cost_per_test": 0.016803, "avg_input_tokens": 1362.4, "avg_output_tokens": 1070.7, "avg_reasoning_tokens": 962.2, "avg_cache_read_tokens": 362.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 16.7, "median_query_seconds": 11.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 92.608, "latency": 17.242, "stderr": 0.977, "cost_per_test": 0.01105, "avg_input_tokens": 1362.4, "avg_output_tokens": 998.6, "avg_reasoning_tokens": 833.9, "avg_cache_read_tokens": 568.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 17.2, "median_query_seconds": 11.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-0709": { "accuracy": 92.608, "latency": 41.881, "stderr": 0.977, "cost_per_test": 0.025477, "avg_input_tokens": 1296.0, "avg_output_tokens": 1439.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 41.9, "median_query_seconds": 18.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 92.469, "latency": 23.454, "stderr": 0.986, "cost_per_test": 0.104728, "avg_input_tokens": 1510.4, "avg_output_tokens": 1094.3, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 23.5, "median_query_seconds": 20.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 92.469, "latency": 37.78, "stderr": 1.851, "cost_per_test": 0.001291, "avg_input_tokens": 1662.6, "avg_output_tokens": 1166.2, "avg_reasoning_tokens": 923.2, "avg_cache_read_tokens": 1032.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 37.8, "median_query_seconds": 22.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 92.329, "latency": 8.652, "stderr": 0.994, "cost_per_test": 0.00127, "avg_input_tokens": 1381.7, "avg_output_tokens": 1231.3, "avg_reasoning_tokens": 889.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 8.7, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 92.329, "latency": 35.357, "stderr": 0.994, "cost_per_test": 0.005165, "avg_input_tokens": 1419.5, "avg_output_tokens": 576.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 35.4, "median_query_seconds": 28.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 92.19, "latency": 14.605, "stderr": 1.002, "cost_per_test": null, "avg_input_tokens": 1384.1, "avg_output_tokens": 1363.8, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 14.6, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 92.05, "latency": 5.274, "stderr": 1.01, "cost_per_test": 0.001275, "avg_input_tokens": 1381.7, "avg_output_tokens": 1224.9, "avg_reasoning_tokens": 880.8, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 5.3, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-4.6": { "accuracy": 92.05, "latency": 29.233, "stderr": 1.01, "cost_per_test": 0.004832, "avg_input_tokens": 1378.6, "avg_output_tokens": 1820.3, "turn_metrics": { "tasks_measured": 716, "avg_turns": 1.0, "avg_query_seconds": 29.2, "median_query_seconds": 20.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 91.911, "latency": 3.454, "stderr": 1.018, "cost_per_test": 0.003016, "avg_input_tokens": 1371.5, "avg_output_tokens": 1041.6, "avg_reasoning_tokens": 846.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 3.5, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 91.911, "latency": 20.622, "stderr": 1.018, "cost_per_test": 0.093755, "avg_input_tokens": 1362.4, "avg_output_tokens": 1222.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 20.6, "median_query_seconds": 14.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 91.771, "latency": 8.521, "stderr": 1.065, "cost_per_test": 0.000309, "avg_input_tokens": 1411.6, "avg_output_tokens": 1212.0, "avg_cache_read_tokens": 1056.7, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 8.5, "median_query_seconds": 4.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 91.492, "latency": 54.143, "stderr": 1.042, "cost_per_test": 0.00481, "avg_input_tokens": 1478.3, "avg_output_tokens": 1357.0, "avg_reasoning_tokens": 1219.7, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 54.1, "median_query_seconds": 18.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 91.353, "latency": 10.41, "stderr": 1.05, "cost_per_test": 0.003454, "avg_input_tokens": 1362.4, "avg_output_tokens": 1556.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 10.4, "median_query_seconds": 8.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 91.353, "latency": 14.45, "stderr": 1.08, "cost_per_test": 0.002026, "avg_input_tokens": 1362.4, "avg_output_tokens": 842.8, "avg_reasoning_tokens": 731.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 14.5, "median_query_seconds": 7.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 91.353, "latency": 28.672, "stderr": 1.05, "cost_per_test": 0.030852, "avg_input_tokens": 1510.4, "avg_output_tokens": 1754.7, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 28.7, "median_query_seconds": 19.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 91.353, "latency": 88.539, "stderr": 1.05, "cost_per_test": 0.020461, "avg_input_tokens": 1421.5, "avg_output_tokens": 3125.8, "avg_reasoning_tokens": 2516.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 88.5, "median_query_seconds": 62.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 91.213, "latency": 7.935, "stderr": 1.057, "cost_per_test": 0.010018, "avg_input_tokens": 1482.4, "avg_output_tokens": 371.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 7.9, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 91.213, "latency": 19.793, "stderr": 1.087, "cost_per_test": 0.00025, "avg_input_tokens": 1321.1, "avg_output_tokens": 353.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1063.1, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 19.8, "median_query_seconds": 15.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 91.213, "latency": 65.509, "stderr": 1.057, "cost_per_test": 0.000683, "avg_input_tokens": 1321.2, "avg_output_tokens": 1529.4, "avg_reasoning_tokens": 1440.1, "avg_cache_read_tokens": 1288.1, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 65.5, "median_query_seconds": 47.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 90.934, "latency": 9.654, "stderr": 1.072, "cost_per_test": 0.00126, "avg_input_tokens": 1296.0, "avg_output_tokens": 1743.1, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 9.7, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 90.795, "latency": 21.636, "stderr": 1.634, "cost_per_test": 0.010148, "avg_input_tokens": 1320.1, "avg_output_tokens": 1549.3, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 21.6, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 90.795, "latency": 42.234, "stderr": 1.15, "cost_per_test": 0.001768, "avg_input_tokens": 1419.5, "avg_output_tokens": 1654.8, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 42.2, "median_query_seconds": 20.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 90.656, "latency": 4.835, "stderr": 1.087, "cost_per_test": 0.001417, "avg_input_tokens": 1406.0, "avg_output_tokens": 2272.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 4.8, "median_query_seconds": 3.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-4.5": { "accuracy": 90.656, "latency": 80.35, "stderr": 1.087, "cost_per_test": 0.007079, "avg_input_tokens": 1376.8, "avg_output_tokens": 2842.4, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 80.4, "median_query_seconds": 56.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 90.516, "latency": 7.739, "stderr": 1.094, "cost_per_test": 0.000294, "avg_input_tokens": 1381.7, "avg_output_tokens": 1571.3, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 7.7, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-4.7": { "accuracy": 90.377, "latency": 64.727, "stderr": 1.101, "cost_per_test": 0.005043, "avg_input_tokens": 1376.5, "avg_output_tokens": 2147.8, "avg_cache_read_tokens": 993.9, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 64.7, "median_query_seconds": 35.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 90.237, "latency": 15.13, "stderr": 1.108, "cost_per_test": 0.009205, "avg_input_tokens": 1362.4, "avg_output_tokens": 1751.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 15.1, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 90.237, "latency": 18.026, "stderr": 1.108, "cost_per_test": 0.000507, "avg_input_tokens": 1658.5, "avg_output_tokens": 1474.2, "avg_reasoning_tokens": 1352.7, "avg_cache_read_tokens": 1006.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 18.0, "median_query_seconds": 11.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 90.237, "latency": 24.131, "stderr": 1.108, "cost_per_test": 0.030189, "avg_input_tokens": 1510.4, "avg_output_tokens": 1710.5, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 24.1, "median_query_seconds": 16.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 90.237, "latency": 77.057, "stderr": 1.195, "cost_per_test": 0.00166, "avg_input_tokens": 1368.9, "avg_output_tokens": 1998.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 77.1, "median_query_seconds": 50.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 90.098, "latency": 18.209, "stderr": 1.115, "cost_per_test": 0.001015, "avg_input_tokens": 1430.4, "avg_output_tokens": 1334.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 18.2, "median_query_seconds": 16.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 90.098, "latency": 23.45, "stderr": 1.115, "cost_per_test": 0.003117, "avg_input_tokens": 1378.6, "avg_output_tokens": 579.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 23.4, "median_query_seconds": 20.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 89.958, "latency": 6.336, "stderr": 1.122, "cost_per_test": 0.000951, "avg_input_tokens": 1296.0, "avg_output_tokens": 1124.3, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 6.3, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 89.958, "latency": 9.333, "stderr": 1.122, "cost_per_test": 0.005755, "avg_input_tokens": 1363.4, "avg_output_tokens": 378.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 9.3, "median_query_seconds": 3.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 89.819, "latency": 6.303, "stderr": 1.129, "cost_per_test": 0.008748, "avg_input_tokens": 1482.4, "avg_output_tokens": 286.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 6.3, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 89.679, "latency": 23.473, "stderr": 1.143, "cost_per_test": 0.001441, "avg_input_tokens": 1320.1, "avg_output_tokens": 281.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 23.5, "median_query_seconds": 19.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 89.4, "latency": 8.895, "stderr": 1.15, "cost_per_test": 0.004693, "avg_input_tokens": 1362.4, "avg_output_tokens": 725.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 8.9, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 89.261, "latency": 30.377, "stderr": 1.169, "cost_per_test": 0.002294, "avg_input_tokens": 1318.3, "avg_output_tokens": 1504.0, "avg_cache_read_tokens": 16.0, "avg_cache_write_tokens": 1302.3, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 30.4, "median_query_seconds": 17.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 88.982, "latency": 4.832, "stderr": 1.169, "cost_per_test": 0.000689, "avg_input_tokens": 1328.2, "avg_output_tokens": 451.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 4.8, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 88.703, "latency": 2.946, "stderr": 1.182, "cost_per_test": 0.000309, "avg_input_tokens": 1381.7, "avg_output_tokens": 427.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 2.9, "median_query_seconds": 1.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 88.703, "latency": 6.711, "stderr": 1.182, "cost_per_test": 0.009595, "avg_input_tokens": 1482.4, "avg_output_tokens": 343.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 6.7, "median_query_seconds": 6.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 88.703, "latency": 28.855, "stderr": 1.189, "cost_per_test": 0.003528, "avg_input_tokens": 1338.3, "avg_output_tokens": 1266.0, "avg_cache_read_tokens": 36.0, "avg_cache_write_tokens": 1302.3, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 28.9, "median_query_seconds": 19.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 88.563, "latency": 2.685, "stderr": 1.189, "cost_per_test": 0.000451, "avg_input_tokens": 1381.7, "avg_output_tokens": 783.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 2.7, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 88.563, "latency": 12.468, "stderr": 1.865, "cost_per_test": 0.001387, "avg_input_tokens": 1371.1, "avg_output_tokens": 467.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 12.5, "median_query_seconds": 11.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 88.563, "latency": 18.955, "stderr": 1.604, "cost_per_test": 0.013073, "avg_input_tokens": 1510.4, "avg_output_tokens": 2312.6, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 19.0, "median_query_seconds": 11.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 88.285, "latency": 3.176, "stderr": 1.201, "cost_per_test": 0.001175, "avg_input_tokens": 1363.4, "avg_output_tokens": 393.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 3.2, "median_query_seconds": 2.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 88.145, "latency": 2.688, "stderr": 1.207, "cost_per_test": 0.0024, "avg_input_tokens": 1381.7, "avg_output_tokens": 134.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 2.7, "median_query_seconds": 2.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 88.145, "latency": 7.182, "stderr": 1.207, "cost_per_test": 0.006568, "avg_input_tokens": 1363.4, "avg_output_tokens": 315.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 7.2, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3": { "accuracy": 88.145, "latency": 14.358, "stderr": 1.207, "cost_per_test": 0.015289, "avg_input_tokens": 1295.0, "avg_output_tokens": 760.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 14.4, "median_query_seconds": 11.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 87.866, "latency": 6.356, "stderr": 1.305, "cost_per_test": 0.001278, "avg_input_tokens": 1371.1, "avg_output_tokens": 364.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 6.4, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 87.448, "latency": 2.646, "stderr": 1.237, "cost_per_test": 0.000211, "avg_input_tokens": 1457.0, "avg_output_tokens": 190.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1169.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 2.6, "median_query_seconds": 2.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 86.89, "latency": 5.551, "stderr": 1.601, "cost_per_test": 0.000242, "avg_input_tokens": 1362.5, "avg_output_tokens": 435.6, "avg_reasoning_tokens": 314.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 5.6, "median_query_seconds": 4.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 86.332, "latency": 17.004, "stderr": 1.283, "cost_per_test": 0.001283, "avg_input_tokens": 1362.4, "avg_output_tokens": 3036.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 17.0, "median_query_seconds": 13.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-2-1212": { "accuracy": 86.053, "latency": 7.621, "stderr": 1.294, "cost_per_test": 0.005695, "avg_input_tokens": 1370.4, "avg_output_tokens": 295.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 7.6, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 86.053, "latency": 36.516, "stderr": 1.326, "cost_per_test": null, "avg_input_tokens": 1400.4, "avg_output_tokens": 1201.8, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 36.5, "median_query_seconds": 30.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 85.495, "latency": 16.683, "stderr": 1.315, "cost_per_test": 0.001331, "avg_input_tokens": 1320.1, "avg_output_tokens": 159.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 16.7, "median_query_seconds": 14.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 85.216, "latency": 4.749, "stderr": 1.326, "cost_per_test": 0.001606, "avg_input_tokens": 1404.4, "avg_output_tokens": 420.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 4.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 84.937, "latency": 5.538, "stderr": 1.336, "cost_per_test": 0.000512, "avg_input_tokens": 1328.2, "avg_output_tokens": 461.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 5.5, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 84.798, "latency": 1.597, "stderr": 1.341, "cost_per_test": 0.001092, "avg_input_tokens": 1416.0, "avg_output_tokens": 1617.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 1.6, "median_query_seconds": 1.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 84.658, "latency": 13.848, "stderr": 1.346, "cost_per_test": 0.00293, "avg_input_tokens": 1371.1, "avg_output_tokens": 1496.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 13.8, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 84.658, "latency": 40.598, "stderr": 1.818, "cost_per_test": 0.000454, "avg_input_tokens": 1477.6, "avg_output_tokens": 1847.5, "avg_reasoning_tokens": 1558.9, "avg_cache_read_tokens": 1255.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 40.6, "median_query_seconds": 23.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 84.379, "latency": 19.51, "stderr": 1.53, "cost_per_test": 0.000667, "avg_input_tokens": 1429.6, "avg_output_tokens": 1894.5, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 19.5, "median_query_seconds": 17.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 84.24, "latency": 18.163, "stderr": 1.366, "cost_per_test": 0.010747, "avg_input_tokens": 1371.1, "avg_output_tokens": 1600.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 18.2, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 84.1, "latency": 8.08, "stderr": 1.468, "cost_per_test": 0.007273, "avg_input_tokens": 1363.4, "avg_output_tokens": 386.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 8.1, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 83.821, "latency": 5.347, "stderr": 1.375, "cost_per_test": 0.004702, "avg_input_tokens": 1609.9, "avg_output_tokens": 247.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 5.3, "median_query_seconds": 4.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 82.706, "latency": 1.504, "stderr": 1.421, "cost_per_test": 0.000164, "avg_input_tokens": 1381.7, "avg_output_tokens": 200.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 1.5, "median_query_seconds": 1.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 81.869, "latency": 5.974, "stderr": 1.439, "cost_per_test": 0.002267, "avg_input_tokens": 1482.4, "avg_output_tokens": 270.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 6.0, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 81.869, "latency": 9.774, "stderr": 1.643, "cost_per_test": 0.007289, "avg_input_tokens": 1402.3, "avg_output_tokens": 378.3, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 9.8, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 81.172, "latency": 2.868, "stderr": 1.46, "cost_per_test": 0.000189, "avg_input_tokens": 1371.1, "avg_output_tokens": 285.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 2.9, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 80.753, "latency": 15.727, "stderr": 1.628, "cost_per_test": null, "avg_input_tokens": 1397.4, "avg_output_tokens": 744.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 15.7, "median_query_seconds": 14.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 80.474, "latency": 1.914, "stderr": 1.5, "cost_per_test": 0.000292, "avg_input_tokens": 1363.4, "avg_output_tokens": 388.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 1.9, "median_query_seconds": 1.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 80.195, "latency": 4.753, "stderr": 1.488, "cost_per_test": 0.000416, "avg_input_tokens": 1363.4, "avg_output_tokens": 352.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 4.8, "median_query_seconds": 4.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 80.056, "latency": 97.439, "stderr": 1.622, "cost_per_test": 0.001191, "avg_input_tokens": 1440.6, "avg_output_tokens": 2566.6, "avg_reasoning_tokens": 2273.8, "avg_cache_read_tokens": 1235.9, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 97.4, "median_query_seconds": 50.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 79.637, "latency": 3.367, "stderr": 1.504, "cost_per_test": 0.000439, "avg_input_tokens": 1371.1, "avg_output_tokens": 275.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 3.4, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 71.13, "latency": 9.478, "stderr": 1.692, "cost_per_test": 0.004323, "avg_input_tokens": 1446.5, "avg_output_tokens": 178.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 9.5, "median_query_seconds": 6.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 64.435, "latency": 90.757, "stderr": 1.788, "cost_per_test": 0.03228, "avg_input_tokens": 1386.6, "avg_output_tokens": 4026.7, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 513, "avg_turns": 1.0, "avg_query_seconds": 90.8, "median_query_seconds": 77.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-r-plus": { "accuracy": 62.483, "latency": 5.069, "stderr": 1.808, "cost_per_test": 0.00615, "avg_input_tokens": 1402.3, "avg_output_tokens": 264.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 5.1, "median_query_seconds": 3.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 25.941, "latency": 2.025, "stderr": 1.637, "cost_per_test": 0.000368, "avg_input_tokens": 1446.5, "avg_output_tokens": 197.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 717, "avg_turns": 1.0, "avg_query_seconds": 2.0, "median_query_seconds": 1.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "business": { "anthropic/claude-fable-5-1": { "accuracy": 95.184, "latency": 14.985, "stderr": 0.762, "cost_per_test": 0.070536, "avg_input_tokens": 1571.0, "avg_output_tokens": 1018.2, "avg_cache_write_tokens": 1567.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 15.0, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 94.55, "latency": 20.041, "stderr": 0.817, "cost_per_test": 0.009893, "avg_input_tokens": 1169.7, "avg_output_tokens": 2219.9, "avg_reasoning_tokens": 2043.6, "avg_cache_read_tokens": 912.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 20.0, "median_query_seconds": 12.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 93.79, "latency": 6.442, "stderr": 0.859, "cost_per_test": 0.019347, "avg_input_tokens": 1054.8, "avg_output_tokens": 2368.7, "avg_reasoning_tokens": 2102.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 6.4, "median_query_seconds": 2.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 93.663, "latency": 19.25, "stderr": 0.867, "cost_per_test": 0.030708, "avg_input_tokens": 1054.8, "avg_output_tokens": 2383.2, "avg_reasoning_tokens": 2034.3, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 19.2, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 93.536, "latency": 7.81, "stderr": 0.875, "cost_per_test": 0.022751, "avg_input_tokens": 1569.0, "avg_output_tokens": 518.3, "avg_reasoning_tokens": 329.0, "avg_cache_read_tokens": 2.0, "avg_cache_write_tokens": 1565.1, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 7.8, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 93.536, "latency": 25.757, "stderr": 0.875, "cost_per_test": 0.014848, "avg_input_tokens": 1069.2, "avg_output_tokens": 1623.3, "avg_reasoning_tokens": 1324.6, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 25.8, "median_query_seconds": 20.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 93.409, "latency": 19.549, "stderr": 0.883, "cost_per_test": 0.023771, "avg_input_tokens": 1054.8, "avg_output_tokens": 1805.1, "avg_reasoning_tokens": 1512.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 19.5, "median_query_seconds": 11.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 93.283, "latency": 11.262, "stderr": 0.891, "cost_per_test": 0.0265, "avg_input_tokens": 1046.1, "avg_output_tokens": 709.0, "avg_reasoning_tokens": 624.2, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 11.3, "median_query_seconds": 4.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 93.283, "latency": 17.416, "stderr": 0.891, "cost_per_test": 0.02368, "avg_input_tokens": 1054.8, "avg_output_tokens": 1797.5, "avg_reasoning_tokens": 1488.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 17.4, "median_query_seconds": 10.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 93.283, "latency": 79.437, "stderr": 0.944, "cost_per_test": 0.050755, "avg_input_tokens": 1574.0, "avg_output_tokens": 1714.6, "avg_cache_write_tokens": 16.5, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 79.4, "median_query_seconds": 44.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 93.283, "latency": 336.555, "stderr": 0.891, "cost_per_test": 0.012806, "avg_input_tokens": 1038.3, "avg_output_tokens": 1974.1, "avg_reasoning_tokens": 1774.4, "avg_cache_read_tokens": 743.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 336.6, "median_query_seconds": 242.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 93.156, "latency": 15.86, "stderr": 0.899, "cost_per_test": 0.054887, "avg_input_tokens": 1569.0, "avg_output_tokens": 783.1, "avg_cache_write_tokens": 16.5, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 15.9, "median_query_seconds": 8.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 92.902, "latency": 51.654, "stderr": 0.914, "cost_per_test": 0.016642, "avg_input_tokens": 1107.2, "avg_output_tokens": 2405.1, "avg_reasoning_tokens": 2229.0, "avg_cache_read_tokens": 1.3, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 51.7, "median_query_seconds": 11.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.3": { "accuracy": 92.902, "latency": 417.861, "stderr": 0.914, "cost_per_test": 0.007727, "avg_input_tokens": 1005.4, "avg_output_tokens": 2927.1, "avg_reasoning_tokens": 2709.1, "avg_cache_read_tokens": 806.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 417.9, "median_query_seconds": 387.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 92.776, "latency": 3.906, "stderr": 0.922, "cost_per_test": 0.010355, "avg_input_tokens": 1054.8, "avg_output_tokens": 1169.7, "avg_reasoning_tokens": 915.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 3.9, "median_query_seconds": 2.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 92.776, "latency": 6.7, "stderr": 0.922, "cost_per_test": 0.012621, "avg_input_tokens": 1054.8, "avg_output_tokens": 1471.8, "avg_reasoning_tokens": 1203.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 6.7, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 92.649, "latency": 16.722, "stderr": 0.929, "cost_per_test": 0.041351, "avg_input_tokens": 1569.0, "avg_output_tokens": 1339.4, "avg_cache_write_tokens": 16.5, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 16.7, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 92.649, "latency": 33.413, "stderr": 0.929, "cost_per_test": 0.012798, "avg_input_tokens": 1038.3, "avg_output_tokens": 1960.8, "avg_reasoning_tokens": 1808.4, "avg_cache_read_tokens": 695.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 33.4, "median_query_seconds": 13.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 92.269, "latency": 31.509, "stderr": 0.951, "cost_per_test": 0.017383, "avg_input_tokens": 1054.8, "avg_output_tokens": 5618.6, "avg_reasoning_tokens": 5260.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 31.5, "median_query_seconds": 16.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 92.269, "latency": 45.785, "stderr": 0.951, "cost_per_test": 0.008359, "avg_input_tokens": 1069.2, "avg_output_tokens": 2608.2, "avg_reasoning_tokens": 2375.7, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 45.8, "median_query_seconds": 38.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 92.142, "latency": 65.495, "stderr": 0.958, "cost_per_test": 0.007925, "avg_input_tokens": 1069.2, "avg_output_tokens": 3123.7, "avg_reasoning_tokens": 2727.8, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 65.5, "median_query_seconds": 51.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k3": { "accuracy": 92.015, "latency": 24.794, "stderr": 0.965, "cost_per_test": 0.019582, "avg_input_tokens": 1110.6, "avg_output_tokens": 1263.2, "avg_reasoning_tokens": 1105.7, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 24.8, "median_query_seconds": 8.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 92.015, "latency": 31.992, "stderr": 0.972, "cost_per_test": 0.00905, "avg_input_tokens": 1012.8, "avg_output_tokens": 2027.6, "avg_reasoning_tokens": 1847.7, "avg_cache_read_tokens": 747.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 32.0, "median_query_seconds": 16.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.5": { "accuracy": 92.015, "latency": 35.966, "stderr": 0.965, "cost_per_test": 0.021982, "avg_input_tokens": 1013.6, "avg_output_tokens": 1052.3, "avg_reasoning_tokens": 931.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 36.0, "median_query_seconds": 31.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 91.888, "latency": 26.047, "stderr": 1.219, "cost_per_test": 0.028754, "avg_input_tokens": 1145.2, "avg_output_tokens": 1687.9, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 26.0, "median_query_seconds": 13.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "meta/muse_spark": { "accuracy": 91.888, "latency": 37.05, "stderr": 0.972, "cost_per_test": 0.001453, "avg_input_tokens": 1067.7, "avg_output_tokens": 2154.4, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 37.0, "median_query_seconds": 29.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 91.888, "latency": 41.95, "stderr": 1.573, "cost_per_test": 0.052095, "avg_input_tokens": 1134.2, "avg_output_tokens": 3246.1, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 42.0, "median_query_seconds": 9.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 91.888, "latency": 47.4, "stderr": 0.972, "cost_per_test": 0.012567, "avg_input_tokens": 1033.9, "avg_output_tokens": 2688.7, "avg_reasoning_tokens": 2425.0, "avg_cache_read_tokens": 595.2, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 47.4, "median_query_seconds": 20.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 91.888, "latency": 399.135, "stderr": 0.972, "cost_per_test": 0.0266, "avg_input_tokens": 1095.2, "avg_output_tokens": 6660.9, "avg_reasoning_tokens": 6511.8, "avg_cache_read_tokens": 958.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 399.1, "median_query_seconds": 189.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 91.762, "latency": 22.267, "stderr": 1.441, "cost_per_test": 0.045262, "avg_input_tokens": 1013.6, "avg_output_tokens": 1494.7, "avg_reasoning_tokens": 1353.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 22.3, "median_query_seconds": 9.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 91.635, "latency": 15.114, "stderr": 0.986, "cost_per_test": 0.002306, "avg_input_tokens": 1095.2, "avg_output_tokens": 1651.5, "avg_reasoning_tokens": 1557.9, "avg_cache_read_tokens": 836.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 15.1, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 91.635, "latency": 34.755, "stderr": 1.012, "cost_per_test": 0.038321, "avg_input_tokens": 1013.6, "avg_output_tokens": 2610.5, "avg_reasoning_tokens": 2451.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 34.8, "median_query_seconds": 11.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 91.635, "latency": 46.752, "stderr": 1.393, "cost_per_test": 0.078843, "avg_input_tokens": 1569.0, "avg_output_tokens": 4864.1, "avg_cache_write_tokens": 1567.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 46.8, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 91.635, "latency": 102.866, "stderr": 0.986, "cost_per_test": 0.01534, "avg_input_tokens": 1032.6, "avg_output_tokens": 3739.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 755.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 102.9, "median_query_seconds": 39.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 91.508, "latency": 73.173, "stderr": 0.992, "cost_per_test": 0.036474, "avg_input_tokens": 1021.6, "avg_output_tokens": 3754.5, "avg_cache_read_tokens": 831.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 73.2, "median_query_seconds": 12.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4-0709": { "accuracy": 91.508, "latency": 124.718, "stderr": 0.992, "cost_per_test": 0.054675, "avg_input_tokens": 999.4, "avg_output_tokens": 3445.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 124.7, "median_query_seconds": 57.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 91.381, "latency": 6.983, "stderr": 0.999, "cost_per_test": 0.000925, "avg_input_tokens": 1054.8, "avg_output_tokens": 2048.3, "avg_reasoning_tokens": 1747.9, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 7.0, "median_query_seconds": 4.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 91.381, "latency": 8.986, "stderr": 0.999, "cost_per_test": 0.011664, "avg_input_tokens": 1121.9, "avg_output_tokens": 1887.2, "avg_reasoning_tokens": 1741.0, "avg_cache_read_tokens": 1056.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 9.0, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 91.381, "latency": 37.273, "stderr": 0.999, "cost_per_test": 0.00085, "avg_input_tokens": 1034.1, "avg_output_tokens": 3236.3, "avg_cache_read_tokens": 607.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 37.3, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 91.381, "latency": 49.206, "stderr": 0.999, "cost_per_test": 0.023832, "avg_input_tokens": 1013.6, "avg_output_tokens": 2256.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 49.2, "median_query_seconds": 17.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 91.128, "latency": 22.186, "stderr": 1.012, "cost_per_test": 0.099964, "avg_input_tokens": 1145.2, "avg_output_tokens": 1103.8, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 22.2, "median_query_seconds": 14.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 91.128, "latency": 57.848, "stderr": 1.012, "cost_per_test": 0.015119, "avg_input_tokens": 1034.1, "avg_output_tokens": 3202.9, "avg_reasoning_tokens": 3000.5, "avg_cache_read_tokens": 370.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 57.8, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.1": { "accuracy": 91.128, "latency": 66.948, "stderr": 1.012, "cost_per_test": 0.012184, "avg_input_tokens": 1027.1, "avg_output_tokens": 3691.5, "avg_reasoning_tokens": 3426.0, "avg_cache_read_tokens": 819.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 66.9, "median_query_seconds": 20.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 91.001, "latency": 15.056, "stderr": 1.086, "cost_per_test": 0.002321, "avg_input_tokens": 1088.3, "avg_output_tokens": 1752.6, "avg_reasoning_tokens": 1655.6, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 15.1, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 91.001, "latency": 29.998, "stderr": 1.019, "cost_per_test": null, "avg_input_tokens": 1085.7, "avg_output_tokens": 2898.9, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 30.0, "median_query_seconds": 9.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 91.001, "latency": 58.163, "stderr": 1.069, "cost_per_test": 0.012305, "avg_input_tokens": 1108.2, "avg_output_tokens": 3021.7, "avg_reasoning_tokens": 2918.0, "avg_cache_read_tokens": 880.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 58.2, "median_query_seconds": 13.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 91.001, "latency": 102.176, "stderr": 1.019, "cost_per_test": 0.024141, "avg_input_tokens": 1054.8, "avg_output_tokens": 3812.5, "avg_reasoning_tokens": 3286.5, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 102.2, "median_query_seconds": 60.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 90.875, "latency": 6.029, "stderr": 1.025, "cost_per_test": 0.006882, "avg_input_tokens": 1013.6, "avg_output_tokens": 404.5, "avg_reasoning_tokens": 324.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 6.0, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 90.748, "latency": 31.453, "stderr": 1.032, "cost_per_test": 0.006461, "avg_input_tokens": 1032.6, "avg_output_tokens": 2072.3, "avg_cache_read_tokens": 750.5, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 31.5, "median_query_seconds": 19.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 90.748, "latency": 91.325, "stderr": 1.032, "cost_per_test": 0.011059, "avg_input_tokens": 1027.2, "avg_output_tokens": 3333.9, "avg_cache_read_tokens": 778.4, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 91.3, "median_query_seconds": 59.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 90.621, "latency": 4.543, "stderr": 1.038, "cost_per_test": 0.004449, "avg_input_tokens": 1054.8, "avg_output_tokens": 1653.2, "avg_reasoning_tokens": 1377.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 4.5, "median_query_seconds": 3.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 90.241, "latency": 76.16, "stderr": 1.057, "cost_per_test": 0.007374, "avg_input_tokens": 1111.2, "avg_output_tokens": 2272.7, "avg_reasoning_tokens": 2099.2, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 76.2, "median_query_seconds": 21.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 90.114, "latency": 38.449, "stderr": 1.069, "cost_per_test": 0.003023, "avg_input_tokens": 1040.9, "avg_output_tokens": 2203.2, "avg_cache_read_tokens": 31.0, "avg_cache_write_tokens": 1007.9, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 38.4, "median_query_seconds": 15.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 90.114, "latency": 97.784, "stderr": 1.063, "cost_per_test": 0.001082, "avg_input_tokens": 1016.2, "avg_output_tokens": 2493.2, "avg_reasoning_tokens": 2378.1, "avg_cache_read_tokens": 956.2, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 97.8, "median_query_seconds": 51.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 90.047, "latency": 0.0, "stderr": 1.03, "cost_per_test": 0.006394, "avg_input_tokens": 1053.3, "avg_output_tokens": 855.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 89.987, "latency": 14.834, "stderr": 1.069, "cost_per_test": 0.011072, "avg_input_tokens": 1013.6, "avg_output_tokens": 1130.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 14.8, "median_query_seconds": 7.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 89.987, "latency": 72.573, "stderr": 1.479, "cost_per_test": 0.002321, "avg_input_tokens": 1295.8, "avg_output_tokens": 2554.8, "avg_reasoning_tokens": 2364.8, "avg_cache_read_tokens": 1072.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 72.6, "median_query_seconds": 26.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 89.861, "latency": 8.597, "stderr": 1.075, "cost_per_test": 0.001312, "avg_input_tokens": 1043.5, "avg_output_tokens": 1492.3, "avg_reasoning_tokens": 1092.7, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 8.6, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 89.861, "latency": 19.818, "stderr": 1.075, "cost_per_test": 0.017077, "avg_input_tokens": 1013.6, "avg_output_tokens": 1581.0, "avg_reasoning_tokens": 1404.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 19.8, "median_query_seconds": 11.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 89.861, "latency": 42.002, "stderr": 1.528, "cost_per_test": 0.045443, "avg_input_tokens": 1134.2, "avg_output_tokens": 2802.7, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 42.0, "median_query_seconds": 9.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 89.861, "latency": 77.044, "stderr": 1.075, "cost_per_test": 0.005233, "avg_input_tokens": 1021.4, "avg_output_tokens": 4276.7, "avg_cache_read_tokens": 854.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 782, "avg_turns": 1.0, "avg_query_seconds": 77.0, "median_query_seconds": 14.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 89.734, "latency": 19.061, "stderr": 1.086, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 89.734, "latency": 20.098, "stderr": 1.098, "cost_per_test": 0.003267, "avg_input_tokens": 1013.6, "avg_output_tokens": 1506.9, "avg_reasoning_tokens": 1371.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 20.1, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 89.354, "latency": 8.321, "stderr": 1.098, "cost_per_test": 0.039973, "avg_input_tokens": 1117.2, "avg_output_tokens": 309.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 8.3, "median_query_seconds": 7.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 89.354, "latency": 26.597, "stderr": 1.098, "cost_per_test": 0.001478, "avg_input_tokens": 1069.2, "avg_output_tokens": 3428.7, "avg_reasoning_tokens": 3035.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 26.6, "median_query_seconds": 21.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 89.227, "latency": 31.297, "stderr": 1.104, "cost_per_test": 0.000852, "avg_input_tokens": 1291.8, "avg_output_tokens": 2920.7, "avg_reasoning_tokens": 2741.4, "avg_cache_read_tokens": 1066.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 31.3, "median_query_seconds": 13.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 89.1, "latency": 14.635, "stderr": 1.109, "cost_per_test": 0.007486, "avg_input_tokens": 1043.5, "avg_output_tokens": 618.2, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 14.6, "median_query_seconds": 11.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 89.1, "latency": 23.018, "stderr": 1.109, "cost_per_test": 0.100986, "avg_input_tokens": 1013.6, "avg_output_tokens": 1429.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 23.0, "median_query_seconds": 14.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 88.973, "latency": 5.683, "stderr": 1.115, "cost_per_test": 0.001306, "avg_input_tokens": 1043.5, "avg_output_tokens": 1501.3, "avg_reasoning_tokens": 1104.3, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 4.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 88.973, "latency": 26.612, "stderr": 1.438, "cost_per_test": 0.011762, "avg_input_tokens": 1015.2, "avg_output_tokens": 1924.4, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 26.6, "median_query_seconds": 15.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 88.847, "latency": 20.02, "stderr": 1.126, "cost_per_test": 0.000872, "avg_input_tokens": 1077.8, "avg_output_tokens": 3814.4, "avg_cache_read_tokens": 799.3, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 20.0, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 88.847, "latency": 27.822, "stderr": 1.121, "cost_per_test": 0.035756, "avg_input_tokens": 1145.2, "avg_output_tokens": 2154.7, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 27.8, "median_query_seconds": 14.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 88.847, "latency": 43.941, "stderr": 1.169, "cost_per_test": 0.003205, "avg_input_tokens": 1023.9, "avg_output_tokens": 2365.3, "avg_cache_read_tokens": 49.5, "avg_cache_write_tokens": 974.4, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 43.9, "median_query_seconds": 15.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 88.72, "latency": 24.113, "stderr": 1.126, "cost_per_test": 0.001053, "avg_input_tokens": 1148.4, "avg_output_tokens": 1926.8, "avg_reasoning_tokens": 1808.6, "avg_cache_read_tokens": 934.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 24.1, "median_query_seconds": 8.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 88.593, "latency": 23.791, "stderr": 1.132, "cost_per_test": 0.006643, "avg_input_tokens": 1013.6, "avg_output_tokens": 3194.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 23.8, "median_query_seconds": 12.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 88.593, "latency": 45.611, "stderr": 1.132, "cost_per_test": 0.005816, "avg_input_tokens": 1052.8, "avg_output_tokens": 758.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 45.6, "median_query_seconds": 25.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-4.6": { "accuracy": 88.466, "latency": 49.227, "stderr": 1.143, "cost_per_test": 0.006347, "avg_input_tokens": 1027.1, "avg_output_tokens": 2604.8, "avg_cache_read_tokens": 749.9, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 49.2, "median_query_seconds": 28.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 88.34, "latency": 30.784, "stderr": 1.164, "cost_per_test": 0.003849, "avg_input_tokens": 1178.9, "avg_output_tokens": 1508.4, "avg_cache_read_tokens": 996.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 30.8, "median_query_seconds": 11.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-4.5": { "accuracy": 88.34, "latency": 219.493, "stderr": 1.143, "cost_per_test": 0.010354, "avg_input_tokens": 1027.1, "avg_output_tokens": 4426.1, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 219.5, "median_query_seconds": 82.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 87.452, "latency": 60.567, "stderr": 1.199, "cost_per_test": 0.002055, "avg_input_tokens": 1081.6, "avg_output_tokens": 3154.2, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 60.6, "median_query_seconds": 46.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 87.199, "latency": 5.263, "stderr": 1.189, "cost_per_test": 0.008106, "avg_input_tokens": 1117.2, "avg_output_tokens": 316.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 5.3, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.7": { "accuracy": 87.199, "latency": 86.161, "stderr": 1.189, "cost_per_test": 0.0068, "avg_input_tokens": 1027.0, "avg_output_tokens": 2998.3, "avg_cache_read_tokens": 699.1, "turn_metrics": { "tasks_measured": 788, "avg_turns": 1.0, "avg_query_seconds": 86.2, "median_query_seconds": 38.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 87.072, "latency": 10.359, "stderr": 1.194, "cost_per_test": 0.005291, "avg_input_tokens": 1013.6, "avg_output_tokens": 949.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 10.4, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 87.072, "latency": 22.144, "stderr": 1.194, "cost_per_test": 0.001932, "avg_input_tokens": 999.4, "avg_output_tokens": 3264.8, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 22.1, "median_query_seconds": 9.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 86.819, "latency": 38.29, "stderr": 1.204, "cost_per_test": 0.035402, "avg_input_tokens": 1144.5, "avg_output_tokens": 2131.2, "turn_metrics": { "tasks_measured": 771, "avg_turns": 1.0, "avg_query_seconds": 38.3, "median_query_seconds": 20.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 86.692, "latency": 24.632, "stderr": 1.209, "cost_per_test": 0.012235, "avg_input_tokens": 1013.6, "avg_output_tokens": 2527.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 24.6, "median_query_seconds": 12.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 86.692, "latency": 43.276, "stderr": 1.256, "cost_per_test": 0.000278, "avg_input_tokens": 1016.3, "avg_output_tokens": 477.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 819.8, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 43.3, "median_query_seconds": 33.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 86.692, "latency": 58.89, "stderr": 1.228, "cost_per_test": 0.002274, "avg_input_tokens": 1052.8, "avg_output_tokens": 2320.4, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 58.9, "median_query_seconds": 28.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 86.565, "latency": 45.357, "stderr": 1.219, "cost_per_test": 0.005087, "avg_input_tokens": 1043.9, "avg_output_tokens": 1961.7, "avg_cache_read_tokens": 36.0, "avg_cache_write_tokens": 1007.9, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 45.4, "median_query_seconds": 22.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 86.565, "latency": 89.584, "stderr": 1.349, "cost_per_test": 0.001802, "avg_input_tokens": 1031.6, "avg_output_tokens": 2604.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 89.6, "median_query_seconds": 39.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 85.678, "latency": 9.108, "stderr": 1.445, "cost_per_test": 0.000415, "avg_input_tokens": 1013.7, "avg_output_tokens": 911.8, "avg_reasoning_tokens": 749.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 9.1, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 85.298, "latency": 5.622, "stderr": 1.261, "cost_per_test": 0.00618, "avg_input_tokens": 1014.6, "avg_output_tokens": 518.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 5.6, "median_query_seconds": 3.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 85.298, "latency": 30.544, "stderr": 1.364, "cost_per_test": 0.179429, "avg_input_tokens": 1146.2, "avg_output_tokens": 2163.1, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 30.5, "median_query_seconds": 12.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 85.044, "latency": 18.53, "stderr": 1.283, "cost_per_test": 0.001292, "avg_input_tokens": 1015.2, "avg_output_tokens": 420.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 18.5, "median_query_seconds": 15.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-3": { "accuracy": 84.791, "latency": 8.804, "stderr": 1.278, "cost_per_test": 0.0131, "avg_input_tokens": 998.4, "avg_output_tokens": 673.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 8.8, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 84.664, "latency": 11.164, "stderr": 1.779, "cost_per_test": 0.001253, "avg_input_tokens": 1072.6, "avg_output_tokens": 478.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 11.2, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 84.537, "latency": 7.615, "stderr": 1.287, "cost_per_test": 0.000997, "avg_input_tokens": 999.4, "avg_output_tokens": 1395.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 7.6, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 84.411, "latency": 8.627, "stderr": 1.291, "cost_per_test": 0.000273, "avg_input_tokens": 1043.5, "avg_output_tokens": 1918.2, "avg_reasoning_tokens": 1496.7, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 8.6, "median_query_seconds": 7.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 84.284, "latency": 13.622, "stderr": 1.3, "cost_per_test": 0.002074, "avg_input_tokens": 1041.6, "avg_output_tokens": 344.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 13.6, "median_query_seconds": 9.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 83.397, "latency": 2.586, "stderr": 1.325, "cost_per_test": 0.000453, "avg_input_tokens": 1043.5, "avg_output_tokens": 871.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 2.6, "median_query_seconds": 1.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 83.27, "latency": 5.569, "stderr": 1.329, "cost_per_test": 0.007379, "avg_input_tokens": 1117.2, "avg_output_tokens": 268.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 5.6, "median_query_seconds": 5.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 83.016, "latency": 4.606, "stderr": 1.337, "cost_per_test": 0.000542, "avg_input_tokens": 1015.7, "avg_output_tokens": 362.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 4.6, "median_query_seconds": 4.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 82.636, "latency": 21.458, "stderr": 1.397, "cost_per_test": null, "avg_input_tokens": 1048.0, "avg_output_tokens": 1018.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 21.5, "median_query_seconds": 20.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 82.51, "latency": 22.853, "stderr": 1.352, "cost_per_test": 0.001792, "avg_input_tokens": 1013.6, "avg_output_tokens": 4354.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 22.9, "median_query_seconds": 15.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 82.129, "latency": 3.578, "stderr": 1.364, "cost_per_test": 0.001132, "avg_input_tokens": 1014.6, "avg_output_tokens": 453.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 3.6, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 82.003, "latency": 19.528, "stderr": 1.702, "cost_per_test": 0.01488, "avg_input_tokens": 1145.2, "avg_output_tokens": 2746.9, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 19.5, "median_query_seconds": 8.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 81.876, "latency": 4.674, "stderr": 1.371, "cost_per_test": 0.000434, "avg_input_tokens": 1043.5, "avg_output_tokens": 823.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 1.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 80.862, "latency": 6.531, "stderr": 1.448, "cost_per_test": 0.04245, "avg_input_tokens": 1117.2, "avg_output_tokens": 342.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 6.5, "median_query_seconds": 6.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 80.735, "latency": 8.464, "stderr": 1.435, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 80.735, "latency": 48.277, "stderr": 1.72, "cost_per_test": 0.000554, "avg_input_tokens": 1106.1, "avg_output_tokens": 2444.4, "avg_reasoning_tokens": 2169.2, "avg_cache_read_tokens": 904.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 48.3, "median_query_seconds": 27.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 80.355, "latency": 3.559, "stderr": 1.414, "cost_per_test": 0.002419, "avg_input_tokens": 1043.5, "avg_output_tokens": 222.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 3.6, "median_query_seconds": 2.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 80.355, "latency": 20.238, "stderr": 1.538, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 79.709, "latency": 44.91, "stderr": 1.213, "cost_per_test": 0.01941, "avg_input_tokens": 1072.5, "avg_output_tokens": 3453.1, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.4, "avg_query_seconds": 44.9, "median_query_seconds": 16.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-2-1212": { "accuracy": 79.214, "latency": 9.121, "stderr": 1.445, "cost_per_test": 0.005639, "avg_input_tokens": 1040.8, "avg_output_tokens": 355.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 9.1, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 79.087, "latency": 7.893, "stderr": 1.448, "cost_per_test": 0.006438, "avg_input_tokens": 1014.6, "avg_output_tokens": 390.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 7.9, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 79.087, "latency": 121.732, "stderr": 1.488, "cost_per_test": 0.001766, "avg_input_tokens": 1069.1, "avg_output_tokens": 4101.3, "avg_reasoning_tokens": 3832.0, "avg_cache_read_tokens": 881.3, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 121.7, "median_query_seconds": 67.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 78.834, "latency": 3.11, "stderr": 1.454, "cost_per_test": 0.000205, "avg_input_tokens": 1160.4, "avg_output_tokens": 218.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 912.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 3.1, "median_query_seconds": 1.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 77.693, "latency": 7.518, "stderr": 1.533, "cost_per_test": 0.006093, "avg_input_tokens": 1014.6, "avg_output_tokens": 355.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 7.5, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 75.665, "latency": 4.955, "stderr": 1.528, "cost_per_test": 0.000424, "avg_input_tokens": 1015.7, "avg_output_tokens": 408.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 5.0, "median_query_seconds": 4.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 75.665, "latency": 29.982, "stderr": 1.546, "cost_per_test": null, "avg_input_tokens": 1048.0, "avg_output_tokens": 1599.8, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 30.0, "median_query_seconds": 19.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 75.539, "latency": 11.181, "stderr": 1.53, "cost_per_test": 0.003029, "avg_input_tokens": 1072.7, "avg_output_tokens": 1661.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 11.2, "median_query_seconds": 7.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 74.271, "latency": 4.416, "stderr": 1.556, "cost_per_test": 0.001053, "avg_input_tokens": 1015.2, "avg_output_tokens": 154.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 4.4, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 73.384, "latency": 3.293, "stderr": 1.573, "cost_per_test": 0.001252, "avg_input_tokens": 1055.0, "avg_output_tokens": 367.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 3.3, "median_query_seconds": 2.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 73.384, "latency": 7.866, "stderr": 1.678, "cost_per_test": 0.006893, "avg_input_tokens": 1056.1, "avg_output_tokens": 425.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 7.9, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 72.877, "latency": 2.094, "stderr": 1.583, "cost_per_test": 0.000895, "avg_input_tokens": 1119.4, "avg_output_tokens": 1342.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 2.1, "median_query_seconds": 1.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 72.37, "latency": 1.992, "stderr": 1.634, "cost_per_test": 0.000272, "avg_input_tokens": 1014.6, "avg_output_tokens": 426.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 2.0, "median_query_seconds": 1.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 71.736, "latency": 7.912, "stderr": 1.603, "cost_per_test": 0.004557, "avg_input_tokens": 1161.2, "avg_output_tokens": 372.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 7.9, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 71.61, "latency": 1.793, "stderr": 1.607, "cost_per_test": 0.000151, "avg_input_tokens": 1043.5, "avg_output_tokens": 243.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 1.8, "median_query_seconds": 1.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 70.596, "latency": 5.154, "stderr": 1.622, "cost_per_test": 0.000479, "avg_input_tokens": 1072.7, "avg_output_tokens": 440.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 5.2, "median_query_seconds": 3.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 70.342, "latency": 5.379, "stderr": 1.626, "cost_per_test": 0.000405, "avg_input_tokens": 1014.6, "avg_output_tokens": 421.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 5.4, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 69.835, "latency": 5.48, "stderr": 1.634, "cost_per_test": 0.001917, "avg_input_tokens": 1117.2, "avg_output_tokens": 255.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 5.5, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 69.328, "latency": 4.059, "stderr": 1.642, "cost_per_test": 0.000198, "avg_input_tokens": 1072.7, "avg_output_tokens": 393.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 4.1, "median_query_seconds": 2.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 69.328, "latency": 7.248, "stderr": 1.642, "cost_per_test": 0.001573, "avg_input_tokens": 1109.4, "avg_output_tokens": 2702.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 7.2, "median_query_seconds": 3.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 64.385, "latency": 101.826, "stderr": 1.705, "cost_per_test": 0.047867, "avg_input_tokens": 1083.9, "avg_output_tokens": 6165.4, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 588, "avg_turns": 1.0, "avg_query_seconds": 101.8, "median_query_seconds": 89.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 47.275, "latency": 9.096, "stderr": 1.774, "cost_per_test": 0.004523, "avg_input_tokens": 1109.1, "avg_output_tokens": 288.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 9.1, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "cohere/command-r-plus": { "accuracy": 41.572, "latency": 5.995, "stderr": 1.754, "cost_per_test": 0.00596, "avg_input_tokens": 1056.1, "avg_output_tokens": 332.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 6.0, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 34.728, "latency": 10.899, "stderr": 1.695, "cost_per_test": 0.008309, "avg_input_tokens": 1112.7, "avg_output_tokens": 331.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 319, "avg_turns": 1.0, "avg_query_seconds": 10.9, "median_query_seconds": 8.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 30.418, "latency": 2.722, "stderr": 1.634, "cost_per_test": 0.000353, "avg_input_tokens": 1109.1, "avg_output_tokens": 327.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 789, "avg_turns": 1.0, "avg_query_seconds": 2.7, "median_query_seconds": 2.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "chemistry": { "anthropic/claude-fable-5": { "accuracy": 93.905, "latency": 39.622, "stderr": 0.711, "cost_per_test": 0.094911, "avg_input_tokens": 2123.7, "avg_output_tokens": 1403.0, "avg_cache_write_tokens": 1409.1, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 39.6, "median_query_seconds": 12.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 93.816, "latency": 18.464, "stderr": 0.716, "cost_per_test": 0.095732, "avg_input_tokens": 2125.7, "avg_output_tokens": 1383.4, "avg_cache_write_tokens": 2121.7, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 18.5, "median_query_seconds": 8.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 93.375, "latency": 22.142, "stderr": 0.744, "cost_per_test": 0.011437, "avg_input_tokens": 1662.2, "avg_output_tokens": 2553.6, "avg_reasoning_tokens": 2248.3, "avg_cache_read_tokens": 1357.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 22.1, "median_query_seconds": 14.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 93.286, "latency": 9.971, "stderr": 0.744, "cost_per_test": 0.031874, "avg_input_tokens": 2123.7, "avg_output_tokens": 744.1, "avg_cache_write_tokens": 2121.7, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 10.0, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 93.286, "latency": 36.067, "stderr": 0.744, "cost_per_test": 0.044854, "avg_input_tokens": 1609.5, "avg_output_tokens": 3469.6, "avg_reasoning_tokens": 2963.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 36.1, "median_query_seconds": 18.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 93.286, "latency": 67.055, "stderr": 1.074, "cost_per_test": 0.077784, "avg_input_tokens": 1801.5, "avg_output_tokens": 4825.3, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 67.1, "median_query_seconds": 13.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 93.286, "latency": 549.138, "stderr": 0.744, "cost_per_test": 0.104175, "avg_input_tokens": 2128.7, "avg_output_tokens": 3674.2, "avg_cache_read_tokens": 8.4, "avg_cache_write_tokens": 1371.2, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 549.1, "median_query_seconds": 422.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 93.11, "latency": 55.325, "stderr": 0.77, "cost_per_test": 0.035628, "avg_input_tokens": 1596.7, "avg_output_tokens": 2702.9, "avg_reasoning_tokens": 2217.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 55.3, "median_query_seconds": 34.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 92.845, "latency": 29.316, "stderr": 0.766, "cost_per_test": 0.081956, "avg_input_tokens": 2123.7, "avg_output_tokens": 2784.7, "avg_cache_write_tokens": 1375.3, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 29.3, "median_query_seconds": 10.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 92.845, "latency": 33.183, "stderr": 0.766, "cost_per_test": 0.053294, "avg_input_tokens": 1597.5, "avg_output_tokens": 4174.9, "avg_reasoning_tokens": 3566.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 33.2, "median_query_seconds": 23.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 92.845, "latency": 45.052, "stderr": 0.766, "cost_per_test": 0.011447, "avg_input_tokens": 1505.1, "avg_output_tokens": 2556.2, "avg_reasoning_tokens": 2228.8, "avg_cache_read_tokens": 1180.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1128, "avg_turns": 1.0, "avg_query_seconds": 45.1, "median_query_seconds": 22.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 92.491, "latency": 8.465, "stderr": 0.783, "cost_per_test": 0.027138, "avg_input_tokens": 1596.7, "avg_output_tokens": 3299.1, "avg_reasoning_tokens": 2882.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 8.5, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 92.491, "latency": 64.407, "stderr": 0.783, "cost_per_test": 0.019826, "avg_input_tokens": 1682.8, "avg_output_tokens": 3040.7, "avg_reasoning_tokens": 2727.0, "avg_cache_read_tokens": 1019.5, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 64.4, "median_query_seconds": 18.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 92.491, "latency": 211.305, "stderr": 0.808, "cost_per_test": 0.02316, "avg_input_tokens": 1549.1, "avg_output_tokens": 5709.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1244.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 211.3, "median_query_seconds": 91.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 92.491, "latency": 588.835, "stderr": 0.787, "cost_per_test": 0.041657, "avg_input_tokens": 1586.9, "avg_output_tokens": 10042.9, "avg_reasoning_tokens": 9785.2, "avg_cache_read_tokens": 162.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 588.8, "median_query_seconds": 566.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 92.314, "latency": 5.24, "stderr": 0.792, "cost_per_test": 0.014862, "avg_input_tokens": 1596.7, "avg_output_tokens": 1662.2, "avg_reasoning_tokens": 1276.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 5.2, "median_query_seconds": 3.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 92.314, "latency": 9.267, "stderr": 0.792, "cost_per_test": 0.018427, "avg_input_tokens": 1596.7, "avg_output_tokens": 2137.6, "avg_reasoning_tokens": 1727.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 9.3, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 92.314, "latency": 40.784, "stderr": 0.792, "cost_per_test": 0.024737, "avg_input_tokens": 1644.8, "avg_output_tokens": 2750.0, "avg_reasoning_tokens": 2208.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 40.8, "median_query_seconds": 31.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 92.226, "latency": 92.131, "stderr": 0.796, "cost_per_test": 0.011862, "avg_input_tokens": 1644.8, "avg_output_tokens": 4668.5, "avg_reasoning_tokens": 4063.9, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 92.1, "median_query_seconds": 77.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 92.049, "latency": 51.733, "stderr": 0.804, "cost_per_test": 0.017551, "avg_input_tokens": 1546.7, "avg_output_tokens": 2701.3, "avg_reasoning_tokens": 2487.7, "avg_cache_read_tokens": 1166.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 51.7, "median_query_seconds": 19.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 91.961, "latency": 14.047, "stderr": 0.808, "cost_per_test": 0.038842, "avg_input_tokens": 1547.0, "avg_output_tokens": 1038.6, "avg_reasoning_tokens": 867.8, "avg_cache_read_tokens": 11.3, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 14.0, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 91.873, "latency": 50.721, "stderr": 0.812, "cost_per_test": 0.02887, "avg_input_tokens": 1599.5, "avg_output_tokens": 9356.6, "avg_reasoning_tokens": 8819.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 50.7, "median_query_seconds": 43.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 91.873, "latency": 73.25, "stderr": 1.369, "cost_per_test": 0.099424, "avg_input_tokens": 1801.5, "avg_output_tokens": 6262.5, "avg_cache_write_tokens": 108.7, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 73.2, "median_query_seconds": 16.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 91.844, "latency": 126.749, "stderr": 0.815, "cost_per_test": 0.02417, "avg_input_tokens": 1559.5, "avg_output_tokens": 7397.4, "avg_reasoning_tokens": 7082.3, "avg_cache_read_tokens": 1326.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1128, "avg_turns": 1.0, "avg_query_seconds": 126.7, "median_query_seconds": 41.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 91.784, "latency": 57.728, "stderr": 0.816, "cost_per_test": 0.029196, "avg_input_tokens": 1626.8, "avg_output_tokens": 1884.6, "avg_reasoning_tokens": 1667.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 57.7, "median_query_seconds": 19.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 91.784, "latency": 115.333, "stderr": 0.816, "cost_per_test": 0.011081, "avg_input_tokens": 1686.8, "avg_output_tokens": 3412.5, "avg_reasoning_tokens": 3104.1, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 115.3, "median_query_seconds": 38.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 91.696, "latency": 48.434, "stderr": 0.82, "cost_per_test": null, "avg_input_tokens": 1605.3, "avg_output_tokens": 4556.7, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 48.4, "median_query_seconds": 16.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 91.696, "latency": 71.74, "stderr": 0.824, "cost_per_test": 0.015709, "avg_input_tokens": 1599.9, "avg_output_tokens": 3887.2, "avg_reasoning_tokens": 3686.8, "avg_cache_read_tokens": 1407.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 71.7, "median_query_seconds": 13.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 91.608, "latency": 75.947, "stderr": 0.824, "cost_per_test": 0.020916, "avg_input_tokens": 1566.5, "avg_output_tokens": 4531.2, "avg_reasoning_tokens": 4089.1, "avg_cache_read_tokens": 1065.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 75.9, "median_query_seconds": 30.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.5": { "accuracy": 91.608, "latency": 273.466, "stderr": 0.824, "cost_per_test": 0.018195, "avg_input_tokens": 1546.7, "avg_output_tokens": 2832.6, "avg_reasoning_tokens": 2543.5, "avg_cache_read_tokens": 1262.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 273.5, "median_query_seconds": 268.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.3": { "accuracy": 91.608, "latency": 307.571, "stderr": 0.824, "cost_per_test": 0.012042, "avg_input_tokens": 1505.1, "avg_output_tokens": 4596.3, "avg_reasoning_tokens": 4284.1, "avg_cache_read_tokens": 1267.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 307.6, "median_query_seconds": 287.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark": { "accuracy": 91.519, "latency": 50.435, "stderr": 0.828, "cost_per_test": 0.001905, "avg_input_tokens": 1559.9, "avg_output_tokens": 2793.6, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 50.4, "median_query_seconds": 38.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 91.431, "latency": 66.874, "stderr": 0.832, "cost_per_test": 0.013391, "avg_input_tokens": 1644.8, "avg_output_tokens": 4189.4, "avg_reasoning_tokens": 3748.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 66.9, "median_query_seconds": 56.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 91.343, "latency": 54.018, "stderr": 0.836, "cost_per_test": 0.004516, "avg_input_tokens": 1535.6, "avg_output_tokens": 3292.6, "avg_cache_read_tokens": 31.0, "avg_cache_write_tokens": 1502.6, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 54.0, "median_query_seconds": 31.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5.3": { "accuracy": 91.343, "latency": 61.175, "stderr": 0.836, "cost_per_test": 0.01539, "avg_input_tokens": 1566.5, "avg_output_tokens": 3167.4, "avg_reasoning_tokens": 2922.6, "avg_cache_read_tokens": 648.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 61.2, "median_query_seconds": 13.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 91.343, "latency": 86.144, "stderr": 0.836, "cost_per_test": 0.046349, "avg_input_tokens": 1555.0, "avg_output_tokens": 4761.6, "avg_cache_read_tokens": 1348.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 86.1, "median_query_seconds": 19.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.5": { "accuracy": 91.254, "latency": 54.56, "stderr": 0.84, "cost_per_test": 0.028489, "avg_input_tokens": 1547.0, "avg_output_tokens": 1328.8, "avg_reasoning_tokens": 1086.0, "avg_cache_read_tokens": 26.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 54.6, "median_query_seconds": 50.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 91.254, "latency": 56.981, "stderr": 0.847, "cost_per_test": 0.011818, "avg_input_tokens": 1548.4, "avg_output_tokens": 3849.9, "avg_cache_read_tokens": 1258.5, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 57.0, "median_query_seconds": 35.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 91.078, "latency": 21.94, "stderr": 1.082, "cost_per_test": 0.048378, "avg_input_tokens": 1547.0, "avg_output_tokens": 1614.1, "avg_reasoning_tokens": 1348.7, "avg_cache_read_tokens": 272.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 21.9, "median_query_seconds": 10.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 91.078, "latency": 30.116, "stderr": 0.847, "cost_per_test": 0.000699, "avg_input_tokens": 1566.5, "avg_output_tokens": 2624.7, "avg_cache_read_tokens": 1238.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 30.1, "median_query_seconds": 8.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 91.078, "latency": 39.771, "stderr": 0.847, "cost_per_test": 0.002165, "avg_input_tokens": 1644.8, "avg_output_tokens": 5001.7, "avg_reasoning_tokens": 4360.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 39.8, "median_query_seconds": 33.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 91.078, "latency": 45.357, "stderr": 1.151, "cost_per_test": 0.055856, "avg_input_tokens": 1812.5, "avg_output_tokens": 3361.2, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 45.4, "median_query_seconds": 21.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 90.989, "latency": 6.086, "stderr": 0.851, "cost_per_test": 0.006593, "avg_input_tokens": 1596.7, "avg_output_tokens": 2445.6, "avg_reasoning_tokens": 2007.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 6.1, "median_query_seconds": 4.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 90.989, "latency": 12.463, "stderr": 0.851, "cost_per_test": 0.00158, "avg_input_tokens": 1596.7, "avg_output_tokens": 3549.7, "avg_reasoning_tokens": 3038.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 12.5, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 90.989, "latency": 15.973, "stderr": 0.935, "cost_per_test": 0.002791, "avg_input_tokens": 1547.0, "avg_output_tokens": 2068.0, "avg_reasoning_tokens": 1890.8, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 16.0, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 90.989, "latency": 18.154, "stderr": 0.851, "cost_per_test": 0.00332, "avg_input_tokens": 1586.9, "avg_output_tokens": 2365.9, "avg_reasoning_tokens": 2194.5, "avg_cache_read_tokens": 1177.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 18.2, "median_query_seconds": 4.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 90.989, "latency": 133.733, "stderr": 0.851, "cost_per_test": 0.034771, "avg_input_tokens": 1636.5, "avg_output_tokens": 5467.9, "avg_reasoning_tokens": 4674.5, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 133.7, "median_query_seconds": 84.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 90.869, "latency": 38.648, "stderr": 0.868, "cost_per_test": 0.18329, "avg_input_tokens": 1812.6, "avg_output_tokens": 2081.4, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 38.6, "median_query_seconds": 23.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 90.813, "latency": 9.099, "stderr": 1.062, "cost_per_test": 0.022172, "avg_input_tokens": 2123.7, "avg_output_tokens": 947.3, "avg_cache_write_tokens": 2121.7, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 9.1, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 90.813, "latency": 47.605, "stderr": 1.205, "cost_per_test": 0.304988, "avg_input_tokens": 1813.5, "avg_output_tokens": 3703.8, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 47.6, "median_query_seconds": 18.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "zai/glm-5-thinking": { "accuracy": 90.813, "latency": 115.706, "stderr": 0.859, "cost_per_test": 0.017438, "avg_input_tokens": 1559.5, "avg_output_tokens": 5285.2, "avg_cache_read_tokens": 1292.3, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 115.7, "median_query_seconds": 72.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 90.724, "latency": 11.56, "stderr": 0.862, "cost_per_test": 0.016085, "avg_input_tokens": 1616.1, "avg_output_tokens": 2554.0, "avg_reasoning_tokens": 2290.8, "avg_cache_read_tokens": 1372.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 11.6, "median_query_seconds": 5.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 90.724, "latency": 35.206, "stderr": 0.873, "cost_per_test": 0.041726, "avg_input_tokens": 1547.0, "avg_output_tokens": 2848.7, "avg_reasoning_tokens": 2586.6, "avg_cache_read_tokens": 548.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1131, "avg_turns": 1.0, "avg_query_seconds": 35.2, "median_query_seconds": 13.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 90.724, "latency": 100.705, "stderr": 0.862, "cost_per_test": 0.006994, "avg_input_tokens": 1555.1, "avg_output_tokens": 5715.8, "avg_cache_read_tokens": 1379.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1122, "avg_turns": 1.0, "avg_query_seconds": 100.7, "median_query_seconds": 21.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 90.548, "latency": 22.87, "stderr": 0.87, "cost_per_test": 0.011033, "avg_input_tokens": 1602.1, "avg_output_tokens": 903.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 22.9, "median_query_seconds": 16.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 90.548, "latency": 50.811, "stderr": 0.912, "cost_per_test": 0.007038, "avg_input_tokens": 1673.6, "avg_output_tokens": 2792.9, "avg_cache_read_tokens": 1393.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 50.8, "median_query_seconds": 16.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 90.459, "latency": 26.979, "stderr": 0.908, "cost_per_test": 0.001114, "avg_input_tokens": 1626.5, "avg_output_tokens": 4848.2, "avg_cache_read_tokens": 1237.4, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 27.0, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 90.459, "latency": 108.606, "stderr": 1.038, "cost_per_test": 0.003601, "avg_input_tokens": 1877.4, "avg_output_tokens": 3715.2, "avg_reasoning_tokens": 3405.1, "avg_cache_read_tokens": 1024.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 108.6, "median_query_seconds": 54.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4-0709": { "accuracy": 90.459, "latency": 135.856, "stderr": 0.873, "cost_per_test": 0.070869, "avg_input_tokens": 1499.1, "avg_output_tokens": 4424.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 135.9, "median_query_seconds": 66.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 90.283, "latency": 0.0, "stderr": 0.88, "cost_per_test": 0.008748, "avg_input_tokens": 1634.5, "avg_output_tokens": 1131.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 90.194, "latency": 7.923, "stderr": 0.884, "cost_per_test": 0.009964, "avg_input_tokens": 1557.0, "avg_output_tokens": 573.4, "avg_reasoning_tokens": 404.3, "avg_cache_read_tokens": 17.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 7.9, "median_query_seconds": 4.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 90.106, "latency": 28.924, "stderr": 0.887, "cost_per_test": 0.024697, "avg_input_tokens": 1551.1, "avg_output_tokens": 2340.5, "avg_reasoning_tokens": 2011.1, "avg_cache_read_tokens": 575.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 28.9, "median_query_seconds": 18.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 89.929, "latency": 48.823, "stderr": 0.894, "cost_per_test": 0.008645, "avg_input_tokens": 1634.5, "avg_output_tokens": 1113.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 48.8, "median_query_seconds": 20.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 89.841, "latency": 13.498, "stderr": 0.898, "cost_per_test": 0.002146, "avg_input_tokens": 1602.1, "avg_output_tokens": 2540.0, "avg_reasoning_tokens": 1874.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 13.5, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 89.841, "latency": 42.513, "stderr": 0.898, "cost_per_test": 0.001299, "avg_input_tokens": 1873.5, "avg_output_tokens": 4198.4, "avg_reasoning_tokens": 3914.6, "avg_cache_read_tokens": 1009.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 42.5, "median_query_seconds": 20.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 89.841, "latency": 43.47, "stderr": 0.898, "cost_per_test": 0.032013, "avg_input_tokens": 1547.0, "avg_output_tokens": 3008.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 43.5, "median_query_seconds": 23.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 89.841, "latency": 127.304, "stderr": 0.898, "cost_per_test": 0.001651, "avg_input_tokens": 1508.3, "avg_output_tokens": 3836.5, "avg_reasoning_tokens": 3650.3, "avg_cache_read_tokens": 1447.1, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 127.3, "median_query_seconds": 65.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 89.664, "latency": 56.436, "stderr": 0.908, "cost_per_test": 0.061301, "avg_input_tokens": 1812.5, "avg_output_tokens": 3724.2, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 56.4, "median_query_seconds": 28.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 89.488, "latency": 8.301, "stderr": 1.108, "cost_per_test": 0.064481, "avg_input_tokens": 1784.5, "avg_output_tokens": 502.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 8.3, "median_query_seconds": 7.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 89.488, "latency": 18.038, "stderr": 0.912, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.7": { "accuracy": 89.223, "latency": 160.189, "stderr": 0.922, "cost_per_test": 0.013335, "avg_input_tokens": 1559.6, "avg_output_tokens": 5869.1, "avg_cache_read_tokens": 1045.5, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 160.2, "median_query_seconds": 71.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 89.134, "latency": 15.401, "stderr": 0.928, "cost_per_test": 0.003744, "avg_input_tokens": 1547.0, "avg_output_tokens": 1684.2, "avg_reasoning_tokens": 1433.1, "avg_cache_read_tokens": 50.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 15.4, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 89.134, "latency": 22.617, "stderr": 0.925, "cost_per_test": 0.016523, "avg_input_tokens": 1547.0, "avg_output_tokens": 1678.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 22.6, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 89.134, "latency": 27.757, "stderr": 0.925, "cost_per_test": 0.002691, "avg_input_tokens": 1499.1, "avg_output_tokens": 4483.2, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 27.8, "median_query_seconds": 18.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-4.6": { "accuracy": 89.134, "latency": 53.391, "stderr": 0.928, "cost_per_test": 0.007878, "avg_input_tokens": 1561.2, "avg_output_tokens": 3161.4, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 53.4, "median_query_seconds": 28.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 89.046, "latency": 9.256, "stderr": 0.928, "cost_per_test": 0.002146, "avg_input_tokens": 1602.1, "avg_output_tokens": 2565.6, "avg_reasoning_tokens": 1899.4, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 9.3, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 89.046, "latency": 21.705, "stderr": 0.928, "cost_per_test": 0.0064, "avg_input_tokens": 1547.0, "avg_output_tokens": 3006.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 21.7, "median_query_seconds": 13.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 88.869, "latency": 66.867, "stderr": 0.938, "cost_per_test": 0.005376, "avg_input_tokens": 1518.6, "avg_output_tokens": 4009.8, "avg_cache_read_tokens": 16.0, "avg_cache_write_tokens": 1502.6, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 66.9, "median_query_seconds": 25.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 88.516, "latency": 44.105, "stderr": 0.948, "cost_per_test": 0.060384, "avg_input_tokens": 1812.5, "avg_output_tokens": 3663.1, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 44.1, "median_query_seconds": 34.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 88.428, "latency": 38.388, "stderr": 1.38, "cost_per_test": 0.017823, "avg_input_tokens": 1506.9, "avg_output_tokens": 2923.9, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 38.4, "median_query_seconds": 23.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 88.163, "latency": 59.35, "stderr": 0.978, "cost_per_test": 0.003506, "avg_input_tokens": 1634.5, "avg_output_tokens": 3575.9, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 59.4, "median_query_seconds": 39.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 87.898, "latency": 11.11, "stderr": 0.969, "cost_per_test": 0.062945, "avg_input_tokens": 1784.5, "avg_output_tokens": 482.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 11.1, "median_query_seconds": 10.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 87.809, "latency": 27.523, "stderr": 0.978, "cost_per_test": 0.001869, "avg_input_tokens": 1615.1, "avg_output_tokens": 2714.1, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 27.5, "median_query_seconds": 23.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 87.721, "latency": 60.449, "stderr": 0.978, "cost_per_test": 0.006569, "avg_input_tokens": 1538.6, "avg_output_tokens": 2502.0, "avg_cache_read_tokens": 36.0, "avg_cache_write_tokens": 1502.6, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 60.4, "median_query_seconds": 33.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 87.633, "latency": 37.247, "stderr": 0.978, "cost_per_test": 0.001572, "avg_input_tokens": 1648.1, "avg_output_tokens": 2892.9, "avg_reasoning_tokens": 2707.7, "avg_cache_read_tokens": 1360.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 37.2, "median_query_seconds": 16.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 87.456, "latency": 14.197, "stderr": 0.984, "cost_per_test": 0.001312, "avg_input_tokens": 1499.1, "avg_output_tokens": 1724.2, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 14.2, "median_query_seconds": 7.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 87.191, "latency": 42.326, "stderr": 1.005, "cost_per_test": 0.000418, "avg_input_tokens": 1507.7, "avg_output_tokens": 749.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1261.4, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 42.3, "median_query_seconds": 29.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 87.102, "latency": 11.67, "stderr": 0.996, "cost_per_test": 0.000436, "avg_input_tokens": 1602.1, "avg_output_tokens": 3278.2, "avg_reasoning_tokens": 2587.7, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 11.7, "median_query_seconds": 9.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 87.014, "latency": 9.297, "stderr": 1.122, "cost_per_test": 0.00054, "avg_input_tokens": 1548.3, "avg_output_tokens": 1156.9, "avg_reasoning_tokens": 851.4, "avg_cache_read_tokens": 1.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 9.3, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3": { "accuracy": 86.749, "latency": 18.732, "stderr": 1.008, "cost_per_test": 0.020273, "avg_input_tokens": 1498.1, "avg_output_tokens": 1051.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 18.7, "median_query_seconds": 13.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 86.749, "latency": 24.487, "stderr": 1.008, "cost_per_test": 0.015137, "avg_input_tokens": 1547.0, "avg_output_tokens": 3053.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 24.5, "median_query_seconds": 14.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 86.661, "latency": 10.034, "stderr": 1.011, "cost_per_test": 0.006322, "avg_input_tokens": 1547.0, "avg_output_tokens": 1050.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 10.0, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 86.661, "latency": 34.973, "stderr": 1.013, "cost_per_test": 0.152426, "avg_input_tokens": 1547.0, "avg_output_tokens": 2153.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 35.0, "median_query_seconds": 21.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 86.307, "latency": 102.14, "stderr": 1.162, "cost_per_test": 0.002417, "avg_input_tokens": 1548.9, "avg_output_tokens": 3341.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 102.1, "median_query_seconds": 49.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-4.5": { "accuracy": 86.131, "latency": 350.263, "stderr": 1.027, "cost_per_test": 0.016233, "avg_input_tokens": 1559.6, "avg_output_tokens": 6953.5, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 350.3, "median_query_seconds": 136.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 85.866, "latency": 6.509, "stderr": 1.035, "cost_per_test": 0.00085, "avg_input_tokens": 1508.2, "avg_output_tokens": 588.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 6.5, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 85.777, "latency": 12.501, "stderr": 1.038, "cost_per_test": 0.013022, "avg_input_tokens": 1784.5, "avg_output_tokens": 511.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 12.5, "median_query_seconds": 10.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 85.424, "latency": 27.441, "stderr": 1.049, "cost_per_test": 0.00228, "avg_input_tokens": 1547.0, "avg_output_tokens": 5507.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 27.4, "median_query_seconds": 20.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 85.336, "latency": 4.106, "stderr": 1.051, "cost_per_test": 0.000798, "avg_input_tokens": 1602.1, "avg_output_tokens": 1594.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 4.1, "median_query_seconds": 2.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 85.247, "latency": 31.291, "stderr": 1.482, "cost_per_test": 0.026179, "avg_input_tokens": 1812.5, "avg_output_tokens": 4873.4, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 31.3, "median_query_seconds": 14.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 84.982, "latency": 15.68, "stderr": 1.375, "cost_per_test": 0.001859, "avg_input_tokens": 1592.2, "avg_output_tokens": 713.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 15.7, "median_query_seconds": 12.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 83.657, "latency": 5.939, "stderr": 1.099, "cost_per_test": 0.001817, "avg_input_tokens": 1548.0, "avg_output_tokens": 748.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 5.9, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 83.127, "latency": 6.964, "stderr": 1.113, "cost_per_test": 0.000652, "avg_input_tokens": 1602.1, "avg_output_tokens": 1228.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 7.0, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 82.951, "latency": 20.885, "stderr": 1.118, "cost_per_test": 0.010238, "avg_input_tokens": 1548.0, "avg_output_tokens": 892.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 20.9, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 82.951, "latency": 47.977, "stderr": 1.14, "cost_per_test": 0.001779, "avg_input_tokens": 1506.9, "avg_output_tokens": 469.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 48.0, "median_query_seconds": 37.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 82.686, "latency": 6.869, "stderr": 1.125, "cost_per_test": 0.012016, "avg_input_tokens": 1784.5, "avg_output_tokens": 444.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 6.9, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 82.067, "latency": 27.216, "stderr": 1.14, "cost_per_test": 0.003645, "avg_input_tokens": 1557.8, "avg_output_tokens": 695.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 27.2, "median_query_seconds": 20.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 80.389, "latency": 31.536, "stderr": 1.306, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 79.77, "latency": 6.262, "stderr": 1.194, "cost_per_test": 0.010358, "avg_input_tokens": 1784.5, "avg_output_tokens": 333.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 6.3, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 79.594, "latency": 15.685, "stderr": 1.204, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 78.975, "latency": 5.361, "stderr": 1.211, "cost_per_test": 0.003754, "avg_input_tokens": 1602.1, "avg_output_tokens": 350.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 5.4, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-2-1212": { "accuracy": 78.71, "latency": 16.668, "stderr": 1.217, "cost_per_test": 0.008818, "avg_input_tokens": 1579.1, "avg_output_tokens": 566.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 16.7, "median_query_seconds": 10.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 77.297, "latency": 4.232, "stderr": 1.245, "cost_per_test": 0.000291, "avg_input_tokens": 1660.1, "avg_output_tokens": 322.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1346.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 4.2, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 76.855, "latency": 10.218, "stderr": 1.254, "cost_per_test": 0.002341, "avg_input_tokens": 1609.1, "avg_output_tokens": 4038.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 10.2, "median_query_seconds": 4.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 75.972, "latency": 17.212, "stderr": 1.27, "cost_per_test": 0.001588, "avg_input_tokens": 1506.9, "avg_output_tokens": 257.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 17.2, "median_query_seconds": 14.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 75.353, "latency": 6.093, "stderr": 1.281, "cost_per_test": 0.000616, "avg_input_tokens": 1508.2, "avg_output_tokens": 583.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 6.1, "median_query_seconds": 5.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 75.265, "latency": 13.454, "stderr": 1.315, "cost_per_test": 0.010192, "avg_input_tokens": 1548.0, "avg_output_tokens": 632.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 13.5, "median_query_seconds": 11.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 73.322, "latency": 14.513, "stderr": 1.315, "cost_per_test": 0.009883, "avg_input_tokens": 1548.0, "avg_output_tokens": 601.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 14.5, "median_query_seconds": 11.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 73.057, "latency": 97.072, "stderr": 1.319, "cost_per_test": 0.0594, "avg_input_tokens": 1605.1, "avg_output_tokens": 7599.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 923, "avg_turns": 1.0, "avg_query_seconds": 97.1, "median_query_seconds": 87.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 71.555, "latency": 2.534, "stderr": 1.341, "cost_per_test": 0.00023, "avg_input_tokens": 1602.1, "avg_output_tokens": 366.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 2.5, "median_query_seconds": 2.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 71.113, "latency": 12.738, "stderr": 1.347, "cost_per_test": 0.007141, "avg_input_tokens": 1712.2, "avg_output_tokens": 619.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 12.7, "median_query_seconds": 10.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 70.848, "latency": 52.813, "stderr": 1.367, "cost_per_test": null, "avg_input_tokens": 1578.6, "avg_output_tokens": 2666.7, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 52.8, "median_query_seconds": 35.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 70.406, "latency": 1.581, "stderr": 1.357, "cost_per_test": 0.001271, "avg_input_tokens": 1619.1, "avg_output_tokens": 1894.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 1.6, "median_query_seconds": 1.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 70.406, "latency": 4.962, "stderr": 1.357, "cost_per_test": 0.001937, "avg_input_tokens": 1585.6, "avg_output_tokens": 615.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 5.0, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 69.611, "latency": 87.951, "stderr": 1.463, "cost_per_test": 0.001183, "avg_input_tokens": 1657.7, "avg_output_tokens": 5436.1, "avg_reasoning_tokens": 5026.8, "avg_cache_read_tokens": 1402.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 88.0, "median_query_seconds": 52.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 68.905, "latency": 17.157, "stderr": 1.404, "cost_per_test": 0.010878, "avg_input_tokens": 1643.3, "avg_output_tokens": 677.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 17.2, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 68.11, "latency": 8.57, "stderr": 1.385, "cost_per_test": 0.000727, "avg_input_tokens": 1592.3, "avg_output_tokens": 680.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 8.6, "median_query_seconds": 5.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 67.226, "latency": 168.987, "stderr": 1.417, "cost_per_test": 0.003468, "avg_input_tokens": 1620.7, "avg_output_tokens": 16017.6, "avg_reasoning_tokens": 7806.8, "avg_cache_read_tokens": 1402.7, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 169.0, "median_query_seconds": 128.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 66.608, "latency": 5.882, "stderr": 1.402, "cost_per_test": 0.000295, "avg_input_tokens": 1592.3, "avg_output_tokens": 584.3, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 5.9, "median_query_seconds": 4.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 66.608, "latency": 22.217, "stderr": 1.402, "cost_per_test": 0.005379, "avg_input_tokens": 1592.3, "avg_output_tokens": 3055.4, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 22.2, "median_query_seconds": 15.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 65.724, "latency": 19.731, "stderr": 1.423, "cost_per_test": null, "avg_input_tokens": 1578.6, "avg_output_tokens": 853.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 19.7, "median_query_seconds": 17.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 64.223, "latency": 3.327, "stderr": 1.44, "cost_per_test": 0.000433, "avg_input_tokens": 1548.0, "avg_output_tokens": 696.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 3.3, "median_query_seconds": 2.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 62.279, "latency": 8.068, "stderr": 1.441, "cost_per_test": 0.000573, "avg_input_tokens": 1548.0, "avg_output_tokens": 567.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 8.1, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 61.219, "latency": 5.68, "stderr": 1.448, "cost_per_test": 0.002622, "avg_input_tokens": 1784.5, "avg_output_tokens": 298.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 50.265, "latency": 18.791, "stderr": 1.486, "cost_per_test": 0.007081, "avg_input_tokens": 1677.1, "avg_output_tokens": 465.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 18.8, "median_query_seconds": 16.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 40.283, "latency": 36.238, "stderr": 1.458, "cost_per_test": 0.018854, "avg_input_tokens": 1592.3, "avg_output_tokens": 3133.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 36.2, "median_query_seconds": 22.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-r-plus": { "accuracy": 28.71, "latency": 6.703, "stderr": 1.345, "cost_per_test": 0.007669, "avg_input_tokens": 1643.3, "avg_output_tokens": 356.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 6.7, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 22.085, "latency": 4.486, "stderr": 1.231, "cost_per_test": 0.000561, "avg_input_tokens": 1677.1, "avg_output_tokens": 563.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1132, "avg_turns": 1.0, "avg_query_seconds": 4.5, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "computer_science": { "anthropic/claude-fable-5-1": { "accuracy": 94.878, "latency": 14.297, "stderr": 1.089, "cost_per_test": 0.072746, "avg_input_tokens": 2138.0, "avg_output_tokens": 920.6, "avg_cache_write_tokens": 2134.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 14.3, "median_query_seconds": 8.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 94.146, "latency": 17.469, "stderr": 1.159, "cost_per_test": 0.08492, "avg_input_tokens": 2136.0, "avg_output_tokens": 1200.5, "avg_cache_read_tokens": 5.4, "avg_cache_write_tokens": 1433.4, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 17.5, "median_query_seconds": 9.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 93.902, "latency": 8.895, "stderr": 1.182, "cost_per_test": 0.028057, "avg_input_tokens": 2136.0, "avg_output_tokens": 588.4, "avg_reasoning_tokens": 344.3, "avg_cache_write_tokens": 2134.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 8.9, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 92.683, "latency": 8.75, "stderr": 1.286, "cost_per_test": 0.022638, "avg_input_tokens": 1580.6, "avg_output_tokens": 2702.3, "avg_reasoning_tokens": 2441.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 8.8, "median_query_seconds": 3.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 92.683, "latency": 19.219, "stderr": 1.286, "cost_per_test": 0.056235, "avg_input_tokens": 2136.0, "avg_output_tokens": 1751.0, "avg_cache_write_tokens": 1424.5, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 19.2, "median_query_seconds": 7.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 92.683, "latency": 21.033, "stderr": 1.286, "cost_per_test": 0.025749, "avg_input_tokens": 1580.6, "avg_output_tokens": 1882.3, "avg_reasoning_tokens": 1563.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 21.0, "median_query_seconds": 11.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 92.683, "latency": 97.967, "stderr": 1.286, "cost_per_test": 0.042102, "avg_input_tokens": 2141.0, "avg_output_tokens": 2306.9, "avg_cache_write_tokens": 1435.2, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 98.0, "median_query_seconds": 78.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 91.951, "latency": 4.823, "stderr": 1.344, "cost_per_test": 0.010676, "avg_input_tokens": 1580.6, "avg_output_tokens": 1107.4, "avg_reasoning_tokens": 876.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 4.8, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 91.707, "latency": 7.006, "stderr": 1.362, "cost_per_test": 0.013111, "avg_input_tokens": 1580.6, "avg_output_tokens": 1432.0, "avg_reasoning_tokens": 1164.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 7.0, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 91.707, "latency": 10.896, "stderr": 1.362, "cost_per_test": 0.022972, "avg_input_tokens": 1580.6, "avg_output_tokens": 1650.9, "avg_reasoning_tokens": 1345.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 10.9, "median_query_seconds": 9.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 91.707, "latency": 39.856, "stderr": 1.362, "cost_per_test": 0.014017, "avg_input_tokens": 1532.8, "avg_output_tokens": 2093.4, "avg_reasoning_tokens": 1927.9, "avg_cache_read_tokens": 1072.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 39.9, "median_query_seconds": 13.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 91.463, "latency": 22.896, "stderr": 1.38, "cost_per_test": 0.035823, "avg_input_tokens": 1580.6, "avg_output_tokens": 2721.8, "avg_reasoning_tokens": 2311.2, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 22.9, "median_query_seconds": 14.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 91.22, "latency": 25.445, "stderr": 1.398, "cost_per_test": 0.020224, "avg_input_tokens": 1577.5, "avg_output_tokens": 1288.3, "avg_reasoning_tokens": 1103.8, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 25.4, "median_query_seconds": 10.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 91.22, "latency": 28.331, "stderr": 1.398, "cost_per_test": 0.016489, "avg_input_tokens": 1589.3, "avg_output_tokens": 1668.8, "avg_reasoning_tokens": 1330.9, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 28.3, "median_query_seconds": 21.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 91.22, "latency": 154.119, "stderr": 1.398, "cost_per_test": 0.017371, "avg_input_tokens": 1499.8, "avg_output_tokens": 4240.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1229.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 154.1, "median_query_seconds": 58.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 90.976, "latency": 6.133, "stderr": 1.415, "cost_per_test": 0.008055, "avg_input_tokens": 1496.2, "avg_output_tokens": 424.2, "avg_reasoning_tokens": 348.0, "avg_cache_read_tokens": 15.6, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 6.1, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 90.976, "latency": 36.425, "stderr": 1.415, "cost_per_test": 0.019599, "avg_input_tokens": 1580.6, "avg_output_tokens": 6270.2, "avg_reasoning_tokens": 5878.6, "avg_cache_read_tokens": 4.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 36.4, "median_query_seconds": 24.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 90.976, "latency": 62.583, "stderr": 1.415, "cost_per_test": 0.008357, "avg_input_tokens": 1589.3, "avg_output_tokens": 3217.2, "avg_reasoning_tokens": 2735.2, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 62.6, "median_query_seconds": 48.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 90.976, "latency": 65.158, "stderr": 1.415, "cost_per_test": 0.017793, "avg_input_tokens": 1627.3, "avg_output_tokens": 2711.6, "avg_reasoning_tokens": 2540.6, "avg_cache_read_tokens": 989.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 65.2, "median_query_seconds": 14.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 90.976, "latency": 495.996, "stderr": 1.415, "cost_per_test": 0.013831, "avg_input_tokens": 1532.8, "avg_output_tokens": 2107.3, "avg_reasoning_tokens": 1890.8, "avg_cache_read_tokens": 1252.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 496.0, "median_query_seconds": 388.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 90.732, "latency": 18.999, "stderr": 1.498, "cost_per_test": 0.009764, "avg_input_tokens": 1655.4, "avg_output_tokens": 2131.9, "avg_reasoning_tokens": 1933.7, "avg_cache_read_tokens": 1241.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 19.0, "median_query_seconds": 13.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 90.732, "latency": 20.3, "stderr": 1.432, "cost_per_test": 0.030318, "avg_input_tokens": 1496.2, "avg_output_tokens": 761.7, "avg_reasoning_tokens": 681.6, "avg_cache_read_tokens": 3.1, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 20.3, "median_query_seconds": 14.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 90.732, "latency": 36.959, "stderr": 1.432, "cost_per_test": 0.009355, "avg_input_tokens": 1498.3, "avg_output_tokens": 2065.4, "avg_reasoning_tokens": 1857.7, "avg_cache_read_tokens": 1156.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 408, "avg_turns": 1.0, "avg_query_seconds": 37.0, "median_query_seconds": 18.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 90.732, "latency": 48.764, "stderr": 2.303, "cost_per_test": 0.052137, "avg_input_tokens": 1688.8, "avg_output_tokens": 3138.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 48.8, "median_query_seconds": 13.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 90.244, "latency": 16.508, "stderr": 1.465, "cost_per_test": 0.002655, "avg_input_tokens": 1598.3, "avg_output_tokens": 1926.3, "avg_reasoning_tokens": 1848.5, "avg_cache_read_tokens": 1388.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 16.5, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 90.244, "latency": 43.156, "stderr": 1.465, "cost_per_test": 0.012207, "avg_input_tokens": 1513.6, "avg_output_tokens": 2461.8, "avg_reasoning_tokens": 2184.8, "avg_cache_read_tokens": 652.6, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 43.2, "median_query_seconds": 20.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 90.0, "latency": 46.177, "stderr": 1.588, "cost_per_test": 0.011516, "avg_input_tokens": 1611.3, "avg_output_tokens": 2775.2, "avg_reasoning_tokens": 2676.2, "avg_cache_read_tokens": 1254.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 46.2, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 90.0, "latency": 49.213, "stderr": 2.382, "cost_per_test": 0.060432, "avg_input_tokens": 1688.8, "avg_output_tokens": 3686.8, "avg_cache_write_tokens": 84.6, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 49.2, "median_query_seconds": 14.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark": { "accuracy": 90.0, "latency": 51.459, "stderr": 1.482, "cost_per_test": 0.001545, "avg_input_tokens": 1553.4, "avg_output_tokens": 2187.1, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 51.5, "median_query_seconds": 43.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 89.756, "latency": 13.524, "stderr": 1.513, "cost_per_test": 0.002273, "avg_input_tokens": 1496.2, "avg_output_tokens": 1645.0, "avg_reasoning_tokens": 1555.2, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 13.5, "median_query_seconds": 5.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 89.756, "latency": 21.483, "stderr": 1.498, "cost_per_test": 0.018843, "avg_input_tokens": 1496.2, "avg_output_tokens": 1735.5, "avg_reasoning_tokens": 1537.4, "avg_cache_read_tokens": 340.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 21.5, "median_query_seconds": 13.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 89.756, "latency": 22.64, "stderr": 2.274, "cost_per_test": 0.045736, "avg_input_tokens": 1496.2, "avg_output_tokens": 1467.0, "avg_reasoning_tokens": 1327.5, "avg_cache_read_tokens": 312.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 22.6, "median_query_seconds": 10.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 89.756, "latency": 31.773, "stderr": 1.498, "cost_per_test": null, "avg_input_tokens": 1582.5, "avg_output_tokens": 2759.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 31.8, "median_query_seconds": 9.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-5.5": { "accuracy": 89.756, "latency": 38.434, "stderr": 1.498, "cost_per_test": 0.027059, "avg_input_tokens": 1562.7, "avg_output_tokens": 1243.8, "avg_reasoning_tokens": 1116.2, "avg_cache_read_tokens": 6.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 38.4, "median_query_seconds": 27.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 89.756, "latency": 44.734, "stderr": 1.498, "cost_per_test": 0.008378, "avg_input_tokens": 1589.3, "avg_output_tokens": 2527.7, "avg_reasoning_tokens": 2254.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 44.7, "median_query_seconds": 37.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 89.512, "latency": 14.003, "stderr": 1.573, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 89.512, "latency": 40.257, "stderr": 2.142, "cost_per_test": 0.043611, "avg_input_tokens": 1699.8, "avg_output_tokens": 2567.4, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 40.3, "median_query_seconds": 19.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 89.487, "latency": 44.147, "stderr": 1.517, "cost_per_test": 0.007687, "avg_input_tokens": 1506.5, "avg_output_tokens": 2240.7, "avg_reasoning_tokens": 1929.5, "avg_cache_read_tokens": 1237.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 409, "avg_turns": 1.0, "avg_query_seconds": 44.1, "median_query_seconds": 21.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 89.024, "latency": 19.153, "stderr": 1.559, "cost_per_test": 0.016148, "avg_input_tokens": 2135.8, "avg_output_tokens": 543.8, "avg_cache_read_tokens": 5.0, "avg_cache_write_tokens": 2128.8, "turn_metrics": { "tasks_measured": 409, "avg_turns": 1.0, "avg_query_seconds": 19.2, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 89.024, "latency": 27.701, "stderr": 1.643, "cost_per_test": 0.129006, "avg_input_tokens": 1699.8, "avg_output_tokens": 1380.1, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 27.7, "median_query_seconds": 19.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 89.024, "latency": 34.835, "stderr": 1.544, "cost_per_test": 0.001091, "avg_input_tokens": 1580.6, "avg_output_tokens": 2331.2, "avg_reasoning_tokens": 1981.1, "turn_metrics": { "tasks_measured": 409, "avg_turns": 1.0, "avg_query_seconds": 34.8, "median_query_seconds": 11.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 89.024, "latency": 39.138, "stderr": 1.544, "cost_per_test": 0.007604, "avg_input_tokens": 1499.5, "avg_output_tokens": 2438.1, "avg_cache_read_tokens": 1220.7, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 39.1, "median_query_seconds": 23.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3": { "accuracy": 89.024, "latency": 43.81, "stderr": 1.544, "cost_per_test": 0.012424, "avg_input_tokens": 1513.6, "avg_output_tokens": 2499.3, "avg_reasoning_tokens": 2253.0, "avg_cache_read_tokens": 606.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 43.8, "median_query_seconds": 10.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.3": { "accuracy": 89.024, "latency": 505.088, "stderr": 1.544, "cost_per_test": 0.00878, "avg_input_tokens": 1479.0, "avg_output_tokens": 3293.0, "avg_reasoning_tokens": 3047.4, "avg_cache_read_tokens": 1239.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 505.1, "median_query_seconds": 473.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 89.024, "latency": 753.864, "stderr": 1.559, "cost_per_test": 0.029423, "avg_input_tokens": 1598.3, "avg_output_tokens": 7321.1, "avg_reasoning_tokens": 7175.6, "avg_cache_read_tokens": 1315.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 753.9, "median_query_seconds": 617.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 88.78, "latency": 41.108, "stderr": 1.559, "cost_per_test": 0.000949, "avg_input_tokens": 1513.6, "avg_output_tokens": 3626.6, "avg_cache_read_tokens": 1192.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 41.1, "median_query_seconds": 7.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 88.78, "latency": 60.904, "stderr": 1.559, "cost_per_test": 0.004781, "avg_input_tokens": 1504.2, "avg_output_tokens": 3873.1, "avg_cache_read_tokens": 1324.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 60.9, "median_query_seconds": 12.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 88.537, "latency": 60.965, "stderr": 1.573, "cost_per_test": 0.030874, "avg_input_tokens": 1504.2, "avg_output_tokens": 3113.7, "avg_cache_read_tokens": 1302.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 61.0, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 88.537, "latency": 104.941, "stderr": 1.573, "cost_per_test": 0.024617, "avg_input_tokens": 1548.9, "avg_output_tokens": 3793.1, "avg_reasoning_tokens": 3261.6, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 104.9, "median_query_seconds": 69.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max": { "accuracy": 88.293, "latency": 0.0, "stderr": 1.588, "cost_per_test": 0.006896, "avg_input_tokens": 1546.9, "avg_output_tokens": 839.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 88.293, "latency": 10.875, "stderr": 1.588, "cost_per_test": 0.054997, "avg_input_tokens": 1671.8, "avg_output_tokens": 398.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 10.9, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 88.293, "latency": 33.394, "stderr": 1.588, "cost_per_test": 0.034376, "avg_input_tokens": 1500.3, "avg_output_tokens": 4575.7, "avg_reasoning_tokens": 2223.9, "avg_cache_read_tokens": 746.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 33.4, "median_query_seconds": 12.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 88.293, "latency": 38.531, "stderr": 1.683, "cost_per_test": 0.004813, "avg_input_tokens": 1669.7, "avg_output_tokens": 1863.8, "avg_cache_read_tokens": 1379.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 38.5, "median_query_seconds": 15.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 88.293, "latency": 174.301, "stderr": 1.588, "cost_per_test": 0.009725, "avg_input_tokens": 1631.3, "avg_output_tokens": 2969.9, "avg_reasoning_tokens": 2824.3, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 174.3, "median_query_seconds": 90.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 88.049, "latency": 58.443, "stderr": 2.268, "cost_per_test": 0.001837, "avg_input_tokens": 1789.9, "avg_output_tokens": 1998.5, "avg_reasoning_tokens": 1821.5, "avg_cache_read_tokens": 1576.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 58.4, "median_query_seconds": 25.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 87.805, "latency": 30.035, "stderr": 1.616, "cost_per_test": 0.021717, "avg_input_tokens": 1496.2, "avg_output_tokens": 1984.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 30.0, "median_query_seconds": 18.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 87.561, "latency": 4.763, "stderr": 1.63, "cost_per_test": 0.004473, "avg_input_tokens": 1580.6, "avg_output_tokens": 1599.5, "avg_reasoning_tokens": 1328.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 4.8, "median_query_seconds": 3.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 87.561, "latency": 10.573, "stderr": 1.63, "cost_per_test": 0.012189, "avg_input_tokens": 1590.0, "avg_output_tokens": 1897.8, "avg_reasoning_tokens": 1663.6, "avg_cache_read_tokens": 1321.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 10.6, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 87.561, "latency": 21.213, "stderr": 1.63, "cost_per_test": 0.000995, "avg_input_tokens": 1622.0, "avg_output_tokens": 1723.2, "avg_reasoning_tokens": 1568.4, "avg_cache_read_tokens": 1273.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 21.2, "median_query_seconds": 9.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 87.561, "latency": 32.653, "stderr": 1.63, "cost_per_test": 0.001708, "avg_input_tokens": 1589.3, "avg_output_tokens": 3872.3, "avg_reasoning_tokens": 3362.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 32.7, "median_query_seconds": 26.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5-thinking": { "accuracy": 87.561, "latency": 75.826, "stderr": 1.63, "cost_per_test": 0.010569, "avg_input_tokens": 1506.5, "avg_output_tokens": 3136.4, "avg_cache_read_tokens": 1152.3, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 75.8, "median_query_seconds": 45.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 87.317, "latency": 7.075, "stderr": 1.643, "cost_per_test": 0.001549, "avg_input_tokens": 1579.1, "avg_output_tokens": 1805.4, "avg_reasoning_tokens": 1375.3, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 7.1, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 87.317, "latency": 18.325, "stderr": 1.643, "cost_per_test": 0.000683, "avg_input_tokens": 1609.9, "avg_output_tokens": 2897.2, "avg_cache_read_tokens": 1253.5, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 18.3, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 87.317, "latency": 87.233, "stderr": 1.643, "cost_per_test": 0.001014, "avg_input_tokens": 1519.4, "avg_output_tokens": 2192.1, "avg_reasoning_tokens": 2100.4, "avg_cache_read_tokens": 1287.5, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 87.2, "median_query_seconds": 50.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 87.073, "latency": 10.423, "stderr": 1.657, "cost_per_test": 0.001574, "avg_input_tokens": 1579.1, "avg_output_tokens": 1807.2, "avg_reasoning_tokens": 1367.2, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 10.4, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 87.073, "latency": 15.171, "stderr": 1.657, "cost_per_test": 0.011691, "avg_input_tokens": 1496.2, "avg_output_tokens": 1087.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 15.2, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 87.073, "latency": 24.636, "stderr": 1.683, "cost_per_test": 0.003661, "avg_input_tokens": 1496.2, "avg_output_tokens": 1645.6, "avg_reasoning_tokens": 1519.3, "avg_cache_read_tokens": 18.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 24.6, "median_query_seconds": 8.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 87.073, "latency": 38.487, "stderr": 1.67, "cost_per_test": 0.003263, "avg_input_tokens": 1531.7, "avg_output_tokens": 2249.7, "avg_cache_read_tokens": 31.0, "avg_cache_write_tokens": 1498.7, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 38.5, "median_query_seconds": 17.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 87.073, "latency": 44.122, "stderr": 1.657, "cost_per_test": 0.006141, "avg_input_tokens": 1546.9, "avg_output_tokens": 714.1, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 44.1, "median_query_seconds": 24.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 86.829, "latency": 8.295, "stderr": 1.67, "cost_per_test": 0.011196, "avg_input_tokens": 1671.8, "avg_output_tokens": 412.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 8.3, "median_query_seconds": 7.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 86.829, "latency": 15.009, "stderr": 1.67, "cost_per_test": 0.004107, "avg_input_tokens": 1496.2, "avg_output_tokens": 1866.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 15.0, "median_query_seconds": 9.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 86.585, "latency": 45.123, "stderr": 1.683, "cost_per_test": 0.000767, "avg_input_tokens": 1785.9, "avg_output_tokens": 2402.7, "avg_reasoning_tokens": 2289.1, "avg_cache_read_tokens": 1134.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 45.1, "median_query_seconds": 26.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4-0709": { "accuracy": 86.585, "latency": 46.115, "stderr": 1.683, "cost_per_test": 0.050302, "avg_input_tokens": 1473.0, "avg_output_tokens": 3058.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 46.1, "median_query_seconds": 19.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-4.7": { "accuracy": 86.585, "latency": 103.442, "stderr": 1.696, "cost_per_test": 0.007504, "avg_input_tokens": 1505.1, "avg_output_tokens": 3283.7, "avg_cache_read_tokens": 1006.9, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 103.4, "median_query_seconds": 35.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-4.6": { "accuracy": 86.341, "latency": 50.687, "stderr": 1.709, "cost_per_test": 0.006772, "avg_input_tokens": 1508.5, "avg_output_tokens": 2666.8, "turn_metrics": { "tasks_measured": 409, "avg_turns": 1.0, "avg_query_seconds": 50.7, "median_query_seconds": 28.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 86.098, "latency": 43.448, "stderr": 1.709, "cost_per_test": 0.00347, "avg_input_tokens": 1514.7, "avg_output_tokens": 2423.1, "avg_cache_read_tokens": 16.0, "avg_cache_write_tokens": 1498.7, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 43.4, "median_query_seconds": 17.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-4.5": { "accuracy": 86.098, "latency": 162.109, "stderr": 1.709, "cost_per_test": 0.009198, "avg_input_tokens": 1506.7, "avg_output_tokens": 3770.2, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 162.1, "median_query_seconds": 63.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 85.854, "latency": 19.727, "stderr": 1.721, "cost_per_test": 0.009532, "avg_input_tokens": 1579.1, "avg_output_tokens": 755.8, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 19.7, "median_query_seconds": 14.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 85.854, "latency": 28.946, "stderr": 1.721, "cost_per_test": 0.119714, "avg_input_tokens": 1496.2, "avg_output_tokens": 1621.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 28.9, "median_query_seconds": 17.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 85.854, "latency": 40.754, "stderr": 1.721, "cost_per_test": 0.005007, "avg_input_tokens": 1534.7, "avg_output_tokens": 1851.8, "avg_cache_read_tokens": 36.0, "avg_cache_write_tokens": 1498.7, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 40.8, "median_query_seconds": 19.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 85.61, "latency": 41.97, "stderr": 2.114, "cost_per_test": 0.237579, "avg_input_tokens": 1700.8, "avg_output_tokens": 2827.6, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 42.0, "median_query_seconds": 17.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 85.61, "latency": 62.738, "stderr": 1.826, "cost_per_test": 0.000259, "avg_input_tokens": 1519.3, "avg_output_tokens": 470.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1442.8, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 62.7, "median_query_seconds": 46.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 85.366, "latency": 26.57, "stderr": 2.298, "cost_per_test": 0.012518, "avg_input_tokens": 1518.3, "avg_output_tokens": 1938.5, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 26.6, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 85.366, "latency": 45.551, "stderr": 1.781, "cost_per_test": 0.002233, "avg_input_tokens": 1546.9, "avg_output_tokens": 2151.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 45.6, "median_query_seconds": 21.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 85.122, "latency": 5.688, "stderr": 1.758, "cost_per_test": 0.00729, "avg_input_tokens": 1497.2, "avg_output_tokens": 536.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 85.122, "latency": 20.726, "stderr": 1.758, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 85.122, "latency": 55.944, "stderr": 1.758, "cost_per_test": 0.052996, "avg_input_tokens": 1700.6, "avg_output_tokens": 3193.0, "turn_metrics": { "tasks_measured": 403, "avg_turns": 1.0, "avg_query_seconds": 55.9, "median_query_seconds": 27.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 84.878, "latency": 11.322, "stderr": 1.769, "cost_per_test": 0.005717, "avg_input_tokens": 1496.2, "avg_output_tokens": 925.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 11.3, "median_query_seconds": 7.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 84.878, "latency": 34.525, "stderr": 1.769, "cost_per_test": 0.044185, "avg_input_tokens": 1699.8, "avg_output_tokens": 2605.7, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 34.5, "median_query_seconds": 20.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 84.634, "latency": 6.365, "stderr": 1.899, "cost_per_test": 0.000369, "avg_input_tokens": 1496.2, "avg_output_tokens": 737.3, "avg_reasoning_tokens": 574.7, "avg_cache_read_tokens": 8.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 6.4, "median_query_seconds": 3.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 84.634, "latency": 18.417, "stderr": 1.781, "cost_per_test": 0.01223, "avg_input_tokens": 1496.2, "avg_output_tokens": 2405.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 18.4, "median_query_seconds": 11.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 84.146, "latency": 8.62, "stderr": 1.975, "cost_per_test": 0.057543, "avg_input_tokens": 1671.8, "avg_output_tokens": 432.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 8.6, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "grok/grok-3": { "accuracy": 84.146, "latency": 10.499, "stderr": 1.804, "cost_per_test": 0.016725, "avg_input_tokens": 1472.0, "avg_output_tokens": 820.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 10.5, "median_query_seconds": 9.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 84.146, "latency": 15.194, "stderr": 1.804, "cost_per_test": 0.001848, "avg_input_tokens": 1473.0, "avg_output_tokens": 2813.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 15.2, "median_query_seconds": 8.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 83.902, "latency": 7.254, "stderr": 1.815, "cost_per_test": 0.001104, "avg_input_tokens": 1473.0, "avg_output_tokens": 1324.5, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 7.3, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 83.902, "latency": 112.665, "stderr": 1.919, "cost_per_test": 0.002135, "avg_input_tokens": 1498.7, "avg_output_tokens": 2823.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 112.7, "median_query_seconds": 52.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 83.659, "latency": 6.094, "stderr": 1.826, "cost_per_test": 0.010214, "avg_input_tokens": 1671.8, "avg_output_tokens": 346.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 6.1, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 83.415, "latency": 17.478, "stderr": 1.837, "cost_per_test": 0.001542, "avg_input_tokens": 1496.2, "avg_output_tokens": 3669.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 17.5, "median_query_seconds": 12.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 83.171, "latency": 11.872, "stderr": 2.247, "cost_per_test": 0.00154, "avg_input_tokens": 1569.5, "avg_output_tokens": 503.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 11.9, "median_query_seconds": 9.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 83.171, "latency": 27.239, "stderr": 1.848, "cost_per_test": 0.003545, "avg_input_tokens": 1508.5, "avg_output_tokens": 679.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 27.2, "median_query_seconds": 19.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 82.927, "latency": 4.076, "stderr": 1.858, "cost_per_test": 0.001393, "avg_input_tokens": 1497.2, "avg_output_tokens": 496.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 4.1, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 82.927, "latency": 50.137, "stderr": 1.858, "cost_per_test": 0.046075, "avg_input_tokens": 1581.5, "avg_output_tokens": 5827.1, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 50.1, "median_query_seconds": 42.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 82.195, "latency": 29.083, "stderr": 1.948, "cost_per_test": 0.001713, "avg_input_tokens": 1518.3, "avg_output_tokens": 385.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 29.1, "median_query_seconds": 25.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 81.951, "latency": 3.595, "stderr": 1.899, "cost_per_test": 0.00063, "avg_input_tokens": 1579.1, "avg_output_tokens": 1179.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 3.6, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 81.951, "latency": 9.315, "stderr": 1.899, "cost_per_test": 0.000345, "avg_input_tokens": 1579.1, "avg_output_tokens": 2332.5, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 9.3, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 81.707, "latency": 28.835, "stderr": 2.398, "cost_per_test": 0.020126, "avg_input_tokens": 1699.8, "avg_output_tokens": 3685.2, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 28.8, "median_query_seconds": 14.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 81.463, "latency": 4.686, "stderr": 1.919, "cost_per_test": 0.000735, "avg_input_tokens": 1501.4, "avg_output_tokens": 460.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 4.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 81.22, "latency": 6.402, "stderr": 1.929, "cost_per_test": 0.009551, "avg_input_tokens": 1671.8, "avg_output_tokens": 302.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 6.4, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 81.22, "latency": 23.628, "stderr": 2.036, "cost_per_test": 0.001057, "avg_input_tokens": 1564.2, "avg_output_tokens": 3157.7, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 23.6, "median_query_seconds": 18.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 80.488, "latency": 6.226, "stderr": 1.957, "cost_per_test": 0.000593, "avg_input_tokens": 1579.1, "avg_output_tokens": 1087.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 6.2, "median_query_seconds": 2.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 80.244, "latency": 7.272, "stderr": 1.966, "cost_per_test": 0.001847, "avg_input_tokens": 1583.0, "avg_output_tokens": 3060.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 7.3, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 78.537, "latency": 4.302, "stderr": 2.052, "cost_per_test": 0.000237, "avg_input_tokens": 1634.0, "avg_output_tokens": 222.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1340.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 4.3, "median_query_seconds": 3.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 78.293, "latency": 8.798, "stderr": 2.044, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 78.049, "latency": 3.692, "stderr": 2.044, "cost_per_test": 0.003045, "avg_input_tokens": 1579.1, "avg_output_tokens": 214.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 3.7, "median_query_seconds": 3.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 78.049, "latency": 10.061, "stderr": 2.044, "cost_per_test": 0.007916, "avg_input_tokens": 1497.2, "avg_output_tokens": 417.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 10.1, "median_query_seconds": 7.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 77.317, "latency": 10.47, "stderr": 2.068, "cost_per_test": 0.001581, "avg_input_tokens": 1518.3, "avg_output_tokens": 238.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 10.5, "median_query_seconds": 8.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 76.341, "latency": 16.809, "stderr": 2.162, "cost_per_test": null, "avg_input_tokens": 1526.0, "avg_output_tokens": 830.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 16.8, "median_query_seconds": 15.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-2-1212": { "accuracy": 76.098, "latency": 10.101, "stderr": 2.106, "cost_per_test": 0.006592, "avg_input_tokens": 1535.6, "avg_output_tokens": 352.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 10.1, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 75.854, "latency": 9.572, "stderr": 2.258, "cost_per_test": 0.008429, "avg_input_tokens": 1497.2, "avg_output_tokens": 468.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 9.6, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 75.61, "latency": 7.355, "stderr": 2.121, "cost_per_test": 0.005473, "avg_input_tokens": 1698.7, "avg_output_tokens": 346.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 7.4, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 74.39, "latency": 13.435, "stderr": 2.156, "cost_per_test": 0.003485, "avg_input_tokens": 1569.5, "avg_output_tokens": 1800.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 13.4, "median_query_seconds": 9.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 74.146, "latency": 34.963, "stderr": 2.162, "cost_per_test": null, "avg_input_tokens": 1526.0, "avg_output_tokens": 1897.7, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 35.0, "median_query_seconds": 20.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 72.927, "latency": 4.266, "stderr": 2.194, "cost_per_test": 0.001692, "avg_input_tokens": 1533.0, "avg_output_tokens": 389.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 4.3, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 72.927, "latency": 44.341, "stderr": 2.342, "cost_per_test": 0.000456, "avg_input_tokens": 1637.0, "avg_output_tokens": 3371.3, "avg_reasoning_tokens": 1555.4, "avg_cache_read_tokens": 1409.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 44.3, "median_query_seconds": 26.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-m.1": { "accuracy": 72.683, "latency": 62.839, "stderr": 2.253, "cost_per_test": 0.001775, "avg_input_tokens": 1600.0, "avg_output_tokens": 4008.5, "avg_reasoning_tokens": 3760.1, "avg_cache_read_tokens": 1480.3, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 62.8, "median_query_seconds": 32.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 71.951, "latency": 3.282, "stderr": 2.219, "cost_per_test": 0.000219, "avg_input_tokens": 1569.5, "avg_output_tokens": 338.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 3.3, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 70.976, "latency": 8.315, "stderr": 2.368, "cost_per_test": 0.008203, "avg_input_tokens": 1585.5, "avg_output_tokens": 423.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 8.3, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 70.0, "latency": 27.638, "stderr": 2.263, "cost_per_test": 0.014657, "avg_input_tokens": 1569.5, "avg_output_tokens": 2303.6, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 27.6, "median_query_seconds": 14.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 69.756, "latency": 4.796, "stderr": 2.268, "cost_per_test": 0.000542, "avg_input_tokens": 1501.4, "avg_output_tokens": 461.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 4.8, "median_query_seconds": 4.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 67.64, "latency": 1.945, "stderr": 2.308, "cost_per_test": 0.001211, "avg_input_tokens": 1594.4, "avg_output_tokens": 1784.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 1.9, "median_query_seconds": 1.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 67.561, "latency": 4.094, "stderr": 2.312, "cost_per_test": 0.000531, "avg_input_tokens": 1569.5, "avg_output_tokens": 361.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 4.1, "median_query_seconds": 3.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 66.098, "latency": 1.769, "stderr": 2.346, "cost_per_test": 0.000191, "avg_input_tokens": 1579.1, "avg_output_tokens": 242.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 1.8, "median_query_seconds": 1.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 65.61, "latency": 5.583, "stderr": 2.346, "cost_per_test": 0.002407, "avg_input_tokens": 1671.8, "avg_output_tokens": 267.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 5.6, "median_query_seconds": 5.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 64.39, "latency": 5.662, "stderr": 2.365, "cost_per_test": 0.000462, "avg_input_tokens": 1497.2, "avg_output_tokens": 396.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 4.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 63.659, "latency": 2.425, "stderr": 2.385, "cost_per_test": 0.000351, "avg_input_tokens": 1497.2, "avg_output_tokens": 502.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 2.4, "median_query_seconds": 1.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 52.683, "latency": 6.983, "stderr": 2.466, "cost_per_test": 0.004989, "avg_input_tokens": 1646.7, "avg_output_tokens": 211.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 7.0, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "cohere/command-r-plus": { "accuracy": 46.585, "latency": 4.718, "stderr": 2.463, "cost_per_test": 0.006329, "avg_input_tokens": 1585.5, "avg_output_tokens": 236.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 28.293, "latency": 2.478, "stderr": 2.224, "cost_per_test": 0.00044, "avg_input_tokens": 1646.7, "avg_output_tokens": 276.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 410, "avg_turns": 1.0, "avg_query_seconds": 2.5, "median_query_seconds": 1.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "economics": { "anthropic/claude-fable-5-1": { "accuracy": 95.142, "latency": 11.834, "stderr": 0.74, "cost_per_test": 0.057274, "avg_input_tokens": 1719.9, "avg_output_tokens": 715.7, "avg_cache_write_tokens": 1715.9, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 11.8, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 93.839, "latency": 12.804, "stderr": 0.828, "cost_per_test": 0.052862, "avg_input_tokens": 1717.9, "avg_output_tokens": 712.8, "avg_cache_write_tokens": 16.7, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 12.8, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 93.72, "latency": 7.939, "stderr": 0.835, "cost_per_test": 0.022734, "avg_input_tokens": 1717.9, "avg_output_tokens": 480.0, "avg_reasoning_tokens": 263.5, "avg_cache_write_tokens": 1715.9, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 7.9, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 93.365, "latency": 14.452, "stderr": 0.857, "cost_per_test": 0.017048, "avg_input_tokens": 1272.4, "avg_output_tokens": 1208.6, "avg_reasoning_tokens": 987.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 14.5, "median_query_seconds": 8.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 92.299, "latency": 609.105, "stderr": 0.955, "cost_per_test": 0.036308, "avg_input_tokens": 1722.9, "avg_output_tokens": 1106.8, "avg_cache_write_tokens": 19.1, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 609.1, "median_query_seconds": 506.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 92.062, "latency": 5.685, "stderr": 0.931, "cost_per_test": 0.016353, "avg_input_tokens": 1272.4, "avg_output_tokens": 1925.9, "avg_reasoning_tokens": 1736.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 91.943, "latency": 8.784, "stderr": 0.937, "cost_per_test": 0.017242, "avg_input_tokens": 1272.4, "avg_output_tokens": 1224.7, "avg_reasoning_tokens": 1008.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 8.8, "median_query_seconds": 6.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 91.943, "latency": 17.613, "stderr": 0.937, "cost_per_test": 0.026377, "avg_input_tokens": 1272.4, "avg_output_tokens": 1986.0, "avg_reasoning_tokens": 1688.9, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 17.6, "median_query_seconds": 12.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 91.825, "latency": 17.061, "stderr": 0.943, "cost_per_test": 0.04071, "avg_input_tokens": 1717.9, "avg_output_tokens": 1284.0, "avg_cache_write_tokens": 16.7, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 17.1, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 91.825, "latency": 68.761, "stderr": 0.943, "cost_per_test": 0.01053, "avg_input_tokens": 1247.8, "avg_output_tokens": 1546.8, "avg_reasoning_tokens": 1405.2, "avg_cache_read_tokens": 831.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 68.8, "median_query_seconds": 11.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 91.706, "latency": 3.142, "stderr": 0.949, "cost_per_test": 0.008178, "avg_input_tokens": 1272.4, "avg_output_tokens": 835.9, "avg_reasoning_tokens": 668.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 3.1, "median_query_seconds": 2.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 91.706, "latency": 20.211, "stderr": 0.949, "cost_per_test": 0.011905, "avg_input_tokens": 1268.0, "avg_output_tokens": 1164.6, "avg_reasoning_tokens": 917.6, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 20.2, "median_query_seconds": 17.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k3": { "accuracy": 91.588, "latency": 22.695, "stderr": 0.955, "cost_per_test": 0.011738, "avg_input_tokens": 1303.8, "avg_output_tokens": 733.0, "avg_reasoning_tokens": 524.9, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 22.7, "median_query_seconds": 14.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.5": { "accuracy": 91.351, "latency": 325.874, "stderr": 0.968, "cost_per_test": 0.009717, "avg_input_tokens": 1247.8, "avg_output_tokens": 1455.1, "avg_reasoning_tokens": 1270.0, "avg_cache_read_tokens": 1006.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 325.9, "median_query_seconds": 365.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 91.232, "latency": 15.915, "stderr": 1.014, "cost_per_test": 0.008745, "avg_input_tokens": 1363.1, "avg_output_tokens": 1941.6, "avg_reasoning_tokens": 1778.2, "avg_cache_read_tokens": 1100.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 15.9, "median_query_seconds": 12.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 91.232, "latency": 29.458, "stderr": 0.985, "cost_per_test": 0.006835, "avg_input_tokens": 1206.1, "avg_output_tokens": 1501.1, "avg_reasoning_tokens": 1324.4, "avg_cache_read_tokens": 956.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 29.5, "median_query_seconds": 15.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 90.995, "latency": 25.017, "stderr": 0.985, "cost_per_test": 0.012976, "avg_input_tokens": 1272.4, "avg_output_tokens": 4113.4, "avg_reasoning_tokens": 3825.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 25.0, "median_query_seconds": 9.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 90.64, "latency": 6.99, "stderr": 1.003, "cost_per_test": 0.018513, "avg_input_tokens": 1216.0, "avg_output_tokens": 414.4, "avg_reasoning_tokens": 353.3, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 7.0, "median_query_seconds": 4.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 90.64, "latency": 24.868, "stderr": 1.528, "cost_per_test": 0.029023, "avg_input_tokens": 1348.1, "avg_output_tokens": 1665.1, "avg_cache_write_tokens": 3.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 24.9, "median_query_seconds": 12.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 90.521, "latency": 5.287, "stderr": 1.008, "cost_per_test": 0.009457, "avg_input_tokens": 1272.4, "avg_output_tokens": 1006.4, "avg_reasoning_tokens": 819.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 5.3, "median_query_seconds": 3.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.2": { "accuracy": 90.403, "latency": 27.0, "stderr": 1.014, "cost_per_test": 0.007413, "avg_input_tokens": 1231.7, "avg_output_tokens": 1472.3, "avg_reasoning_tokens": 1192.9, "avg_cache_read_tokens": 692.8, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 27.0, "median_query_seconds": 14.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 90.284, "latency": 21.06, "stderr": 1.019, "cost_per_test": 0.000508, "avg_input_tokens": 1231.7, "avg_output_tokens": 1807.9, "avg_cache_read_tokens": 608.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 21.1, "median_query_seconds": 6.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 90.284, "latency": 21.494, "stderr": 1.684, "cost_per_test": 0.022582, "avg_input_tokens": 1348.1, "avg_output_tokens": 1235.9, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 21.5, "median_query_seconds": 12.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 90.284, "latency": 21.514, "stderr": 1.019, "cost_per_test": 0.001944, "avg_input_tokens": 1240.4, "avg_output_tokens": 1241.2, "avg_cache_read_tokens": 31.0, "avg_cache_write_tokens": 1207.4, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 21.5, "median_query_seconds": 13.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 90.166, "latency": 25.255, "stderr": 1.305, "cost_per_test": 0.024978, "avg_input_tokens": 1359.1, "avg_output_tokens": 1393.4, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 25.3, "median_query_seconds": 17.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 90.166, "latency": 413.591, "stderr": 1.025, "cost_per_test": 0.005946, "avg_input_tokens": 1212.4, "avg_output_tokens": 2197.8, "avg_reasoning_tokens": 1978.5, "avg_cache_read_tokens": 1013.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 413.6, "median_query_seconds": 399.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 90.071, "latency": 19.787, "stderr": 1.028, "cost_per_test": 0.08772, "avg_input_tokens": 1359.1, "avg_output_tokens": 897.8, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 19.8, "median_query_seconds": 18.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 89.929, "latency": 28.846, "stderr": 1.036, "cost_per_test": 0.01703, "avg_input_tokens": 1216.0, "avg_output_tokens": 743.4, "avg_reasoning_tokens": 659.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 28.8, "median_query_seconds": 22.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 89.929, "latency": 31.834, "stderr": 1.036, "cost_per_test": 0.008894, "avg_input_tokens": 1306.0, "avg_output_tokens": 1344.9, "avg_reasoning_tokens": 1225.6, "avg_cache_read_tokens": 1021.6, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 31.8, "median_query_seconds": 9.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 89.929, "latency": 82.769, "stderr": 1.036, "cost_per_test": 0.01056, "avg_input_tokens": 1225.8, "avg_output_tokens": 2541.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 976.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 82.8, "median_query_seconds": 41.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 89.81, "latency": 3.282, "stderr": 1.041, "cost_per_test": 0.00297, "avg_input_tokens": 1272.4, "avg_output_tokens": 1035.2, "avg_reasoning_tokens": 850.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 3.3, "median_query_seconds": 2.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark": { "accuracy": 89.81, "latency": 34.11, "stderr": 1.041, "cost_per_test": 0.001362, "avg_input_tokens": 1261.0, "avg_output_tokens": 1958.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 34.1, "median_query_seconds": 32.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling": { "accuracy": 89.81, "latency": 40.923, "stderr": 1.041, "cost_per_test": 0.020592, "avg_input_tokens": 1224.0, "avg_output_tokens": 2055.5, "avg_cache_read_tokens": 1077.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 40.9, "median_query_seconds": 11.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 89.692, "latency": 6.504, "stderr": 1.047, "cost_per_test": 0.008769, "avg_input_tokens": 1323.4, "avg_output_tokens": 1366.0, "avg_reasoning_tokens": 1181.4, "avg_cache_read_tokens": 1152.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 6.5, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 89.692, "latency": 9.348, "stderr": 1.047, "cost_per_test": 0.043404, "avg_input_tokens": 1331.1, "avg_output_tokens": 312.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 9.3, "median_query_seconds": 9.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 89.692, "latency": 15.82, "stderr": 1.356, "cost_per_test": 0.030999, "avg_input_tokens": 1216.0, "avg_output_tokens": 976.5, "avg_reasoning_tokens": 887.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 15.8, "median_query_seconds": 8.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 89.692, "latency": 35.519, "stderr": 1.047, "cost_per_test": 0.006698, "avg_input_tokens": 1268.0, "avg_output_tokens": 2021.4, "avg_reasoning_tokens": 1840.1, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 35.5, "median_query_seconds": 32.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 89.692, "latency": 36.032, "stderr": 1.047, "cost_per_test": 0.002849, "avg_input_tokens": 1224.0, "avg_output_tokens": 2288.2, "avg_cache_read_tokens": 1102.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 36.0, "median_query_seconds": 9.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 89.692, "latency": 65.942, "stderr": 1.047, "cost_per_test": 0.006729, "avg_input_tokens": 1268.0, "avg_output_tokens": 2592.6, "avg_reasoning_tokens": 2254.4, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 65.9, "median_query_seconds": 48.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 89.573, "latency": 7.509, "stderr": 1.068, "cost_per_test": 0.044699, "avg_input_tokens": 1331.1, "avg_output_tokens": 329.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 7.5, "median_query_seconds": 7.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 89.573, "latency": 18.946, "stderr": 1.052, "cost_per_test": 0.014362, "avg_input_tokens": 1216.0, "avg_output_tokens": 1284.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 18.9, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 89.573, "latency": 20.197, "stderr": 1.062, "cost_per_test": 0.020471, "avg_input_tokens": 1216.0, "avg_output_tokens": 1375.7, "avg_reasoning_tokens": 1273.9, "avg_cache_read_tokens": 582.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 20.2, "median_query_seconds": 11.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 89.455, "latency": 18.5, "stderr": 1.057, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 89.455, "latency": 27.727, "stderr": 1.189, "cost_per_test": 0.045893, "avg_input_tokens": 1717.9, "avg_output_tokens": 2630.2, "avg_cache_write_tokens": 1715.9, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 27.7, "median_query_seconds": 8.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 89.336, "latency": 6.013, "stderr": 1.062, "cost_per_test": 0.000798, "avg_input_tokens": 1272.4, "avg_output_tokens": 1676.0, "avg_reasoning_tokens": 1386.9, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 6.0, "median_query_seconds": 4.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 89.336, "latency": 8.395, "stderr": 1.062, "cost_per_test": 0.001361, "avg_input_tokens": 1297.2, "avg_output_tokens": 912.4, "avg_reasoning_tokens": 851.7, "avg_cache_read_tokens": 972.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 8.4, "median_query_seconds": 2.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 89.336, "latency": 24.247, "stderr": 1.062, "cost_per_test": 0.001311, "avg_input_tokens": 1268.0, "avg_output_tokens": 2959.6, "avg_reasoning_tokens": 2586.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 24.2, "median_query_seconds": 19.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 89.218, "latency": 38.278, "stderr": 1.088, "cost_per_test": 0.007669, "avg_input_tokens": 1310.2, "avg_output_tokens": 1870.7, "avg_reasoning_tokens": 1795.9, "avg_cache_read_tokens": 1150.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 38.3, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 88.981, "latency": 4.591, "stderr": 1.078, "cost_per_test": 0.005342, "avg_input_tokens": 1216.0, "avg_output_tokens": 242.5, "avg_reasoning_tokens": 185.5, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 4.6, "median_query_seconds": 3.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 88.981, "latency": 9.432, "stderr": 1.078, "cost_per_test": 0.007898, "avg_input_tokens": 1216.0, "avg_output_tokens": 683.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 9.4, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 88.981, "latency": 18.018, "stderr": 1.078, "cost_per_test": 0.010654, "avg_input_tokens": 1216.0, "avg_output_tokens": 962.0, "avg_reasoning_tokens": 825.4, "avg_cache_read_tokens": 431.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 18.0, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 88.981, "latency": 24.478, "stderr": 1.312, "cost_per_test": 0.130533, "avg_input_tokens": 1360.1, "avg_output_tokens": 1468.4, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 24.5, "median_query_seconds": 16.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 88.981, "latency": 28.009, "stderr": 1.078, "cost_per_test": 0.00488, "avg_input_tokens": 1224.7, "avg_output_tokens": 1393.7, "avg_reasoning_tokens": 1158.2, "avg_cache_read_tokens": 1005.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 28.0, "median_query_seconds": 14.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3": { "accuracy": 88.981, "latency": 39.042, "stderr": 1.078, "cost_per_test": 0.009673, "avg_input_tokens": 1231.7, "avg_output_tokens": 1957.8, "avg_reasoning_tokens": 1746.5, "avg_cache_read_tokens": 584.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 39.0, "median_query_seconds": 9.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 88.863, "latency": 24.636, "stderr": 1.083, "cost_per_test": 0.004975, "avg_input_tokens": 1225.8, "avg_output_tokens": 1579.7, "avg_cache_read_tokens": 999.7, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 24.6, "median_query_seconds": 17.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 88.863, "latency": 373.31, "stderr": 1.083, "cost_per_test": 0.014327, "avg_input_tokens": 1297.2, "avg_output_tokens": 3339.4, "avg_reasoning_tokens": 3230.0, "avg_cache_read_tokens": 477.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 373.3, "median_query_seconds": 353.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 88.744, "latency": 15.064, "stderr": 1.093, "cost_per_test": 0.00222, "avg_input_tokens": 1216.0, "avg_output_tokens": 957.9, "avg_reasoning_tokens": 864.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 15.1, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-0709": { "accuracy": 88.744, "latency": 49.714, "stderr": 1.088, "cost_per_test": 0.028301, "avg_input_tokens": 1206.4, "avg_output_tokens": 1645.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 49.7, "median_query_seconds": 19.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 88.507, "latency": 16.623, "stderr": 1.098, "cost_per_test": null, "avg_input_tokens": 1266.6, "avg_output_tokens": 1531.8, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 16.6, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 88.507, "latency": 62.365, "stderr": 1.098, "cost_per_test": 0.000679, "avg_input_tokens": 1218.5, "avg_output_tokens": 1512.4, "avg_reasoning_tokens": 1425.1, "avg_cache_read_tokens": 1155.8, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 62.4, "median_query_seconds": 44.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 88.152, "latency": 10.177, "stderr": 1.112, "cost_per_test": 0.008757, "avg_input_tokens": 1331.1, "avg_output_tokens": 317.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 10.2, "median_query_seconds": 7.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 88.152, "latency": 15.205, "stderr": 1.112, "cost_per_test": 0.005109, "avg_input_tokens": 1310.0, "avg_output_tokens": 1484.8, "avg_reasoning_tokens": 1353.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 15.2, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 88.033, "latency": 11.143, "stderr": 1.141, "cost_per_test": 0.001524, "avg_input_tokens": 1216.0, "avg_output_tokens": 1067.8, "avg_reasoning_tokens": 994.8, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 11.1, "median_query_seconds": 5.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 87.915, "latency": 24.193, "stderr": 1.136, "cost_per_test": 0.002542, "avg_input_tokens": 1378.4, "avg_output_tokens": 941.4, "avg_cache_read_tokens": 1134.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 24.2, "median_query_seconds": 17.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 87.915, "latency": 42.503, "stderr": 1.675, "cost_per_test": 0.001218, "avg_input_tokens": 1493.0, "avg_output_tokens": 1152.9, "avg_reasoning_tokens": 965.8, "avg_cache_read_tokens": 999.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 42.5, "median_query_seconds": 21.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5-thinking": { "accuracy": 87.915, "latency": 71.636, "stderr": 1.122, "cost_per_test": 0.007021, "avg_input_tokens": 1224.6, "avg_output_tokens": 2057.6, "avg_cache_read_tokens": 974.2, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 71.6, "median_query_seconds": 55.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 87.796, "latency": 13.595, "stderr": 1.127, "cost_per_test": 0.007895, "avg_input_tokens": 1271.9, "avg_output_tokens": 630.5, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 13.6, "median_query_seconds": 11.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 87.796, "latency": 83.049, "stderr": 1.127, "cost_per_test": 0.019045, "avg_input_tokens": 1252.0, "avg_output_tokens": 2923.7, "avg_reasoning_tokens": 2470.5, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 83.0, "median_query_seconds": 56.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 87.678, "latency": 13.023, "stderr": 1.131, "cost_per_test": 0.000626, "avg_input_tokens": 1355.4, "avg_output_tokens": 1045.6, "avg_reasoning_tokens": 929.7, "avg_cache_read_tokens": 1120.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 13.0, "median_query_seconds": 7.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 87.678, "latency": 21.108, "stderr": 1.474, "cost_per_test": 0.009453, "avg_input_tokens": 1217.3, "avg_output_tokens": 1446.2, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 21.1, "median_query_seconds": 11.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 87.678, "latency": 32.561, "stderr": 1.131, "cost_per_test": 0.027652, "avg_input_tokens": 1359.2, "avg_output_tokens": 1571.6, "turn_metrics": { "tasks_measured": 837, "avg_turns": 1.0, "avg_query_seconds": 32.6, "median_query_seconds": 23.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 87.441, "latency": 5.184, "stderr": 1.141, "cost_per_test": 0.001202, "avg_input_tokens": 1271.9, "avg_output_tokens": 1193.4, "avg_reasoning_tokens": 865.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 5.2, "median_query_seconds": 4.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 87.441, "latency": 8.176, "stderr": 1.141, "cost_per_test": 0.001203, "avg_input_tokens": 1271.9, "avg_output_tokens": 1167.1, "avg_reasoning_tokens": 838.4, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 8.2, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max": { "accuracy": 87.322, "latency": 0.0, "stderr": 1.145, "cost_per_test": 0.00436, "avg_input_tokens": 1250.0, "avg_output_tokens": 476.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 87.322, "latency": 32.749, "stderr": 1.145, "cost_per_test": 0.004664, "avg_input_tokens": 1250.0, "avg_output_tokens": 527.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 32.7, "median_query_seconds": 25.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 87.204, "latency": 20.018, "stderr": 1.15, "cost_per_test": 0.088583, "avg_input_tokens": 1216.0, "avg_output_tokens": 1172.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 20.0, "median_query_seconds": 14.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.5": { "accuracy": 86.967, "latency": 76.691, "stderr": 1.159, "cost_per_test": 0.006371, "avg_input_tokens": 1224.7, "avg_output_tokens": 2561.8, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 76.7, "median_query_seconds": 55.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-4.7": { "accuracy": 86.848, "latency": 64.194, "stderr": 1.163, "cost_per_test": 0.00463, "avg_input_tokens": 1224.6, "avg_output_tokens": 1983.9, "avg_cache_read_tokens": 901.5, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 64.2, "median_query_seconds": 36.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 86.73, "latency": 11.729, "stderr": 1.168, "cost_per_test": 0.000414, "avg_input_tokens": 1280.8, "avg_output_tokens": 1728.5, "avg_cache_read_tokens": 1036.4, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 11.7, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 86.73, "latency": 18.403, "stderr": 1.177, "cost_per_test": 0.000494, "avg_input_tokens": 1489.0, "avg_output_tokens": 1511.7, "avg_reasoning_tokens": 1401.3, "avg_cache_read_tokens": 1006.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 18.4, "median_query_seconds": 10.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 86.73, "latency": 19.105, "stderr": 1.288, "cost_per_test": 0.00021, "avg_input_tokens": 1218.2, "avg_output_tokens": 306.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1029.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 19.1, "median_query_seconds": 16.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 86.73, "latency": 27.505, "stderr": 1.168, "cost_per_test": 0.00379, "avg_input_tokens": 1216.0, "avg_output_tokens": 1761.0, "avg_reasoning_tokens": 1650.3, "avg_cache_read_tokens": 160.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 27.5, "median_query_seconds": 22.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 86.493, "latency": 21.682, "stderr": 1.177, "cost_per_test": 0.026329, "avg_input_tokens": 1359.1, "avg_output_tokens": 1483.4, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 21.7, "median_query_seconds": 15.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.6": { "accuracy": 86.493, "latency": 36.579, "stderr": 1.177, "cost_per_test": 0.005003, "avg_input_tokens": 1226.4, "avg_output_tokens": 1939.6, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 36.6, "median_query_seconds": 25.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 86.256, "latency": 47.446, "stderr": 1.223, "cost_per_test": 0.001695, "avg_input_tokens": 1250.0, "avg_output_tokens": 1613.4, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 47.4, "median_query_seconds": 21.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 86.019, "latency": 5.183, "stderr": 1.194, "cost_per_test": 0.000626, "avg_input_tokens": 1209.1, "avg_output_tokens": 408.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 5.2, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 86.019, "latency": 5.216, "stderr": 1.194, "cost_per_test": 0.004358, "avg_input_tokens": 1217.0, "avg_output_tokens": 240.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 5.2, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 86.019, "latency": 8.779, "stderr": 1.194, "cost_per_test": 0.001172, "avg_input_tokens": 1206.4, "avg_output_tokens": 1619.7, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 8.8, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 85.9, "latency": 5.639, "stderr": 1.198, "cost_per_test": 0.008379, "avg_input_tokens": 1331.1, "avg_output_tokens": 292.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 5.6, "median_query_seconds": 5.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 85.782, "latency": 11.545, "stderr": 1.662, "cost_per_test": 0.001255, "avg_input_tokens": 1253.6, "avg_output_tokens": 418.5, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 11.5, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 85.308, "latency": 5.538, "stderr": 1.219, "cost_per_test": 0.000863, "avg_input_tokens": 1206.4, "avg_output_tokens": 1001.8, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 5.5, "median_query_seconds": 5.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 85.308, "latency": 6.938, "stderr": 1.219, "cost_per_test": 0.004229, "avg_input_tokens": 1216.0, "avg_output_tokens": 657.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 6.9, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 85.308, "latency": 15.862, "stderr": 1.219, "cost_per_test": 0.002448, "avg_input_tokens": 1234.8, "avg_output_tokens": 404.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 15.9, "median_query_seconds": 13.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 85.308, "latency": 16.328, "stderr": 1.715, "cost_per_test": 0.010968, "avg_input_tokens": 1359.1, "avg_output_tokens": 1921.8, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 16.3, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 85.308, "latency": 29.01, "stderr": 1.223, "cost_per_test": 0.003273, "avg_input_tokens": 1243.4, "avg_output_tokens": 1174.8, "avg_cache_read_tokens": 36.0, "avg_cache_write_tokens": 1207.4, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 29.0, "median_query_seconds": 20.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 85.071, "latency": 15.538, "stderr": 1.227, "cost_per_test": 0.009259, "avg_input_tokens": 1216.0, "avg_output_tokens": 1800.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 15.5, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 85.071, "latency": 71.495, "stderr": 1.329, "cost_per_test": 0.001543, "avg_input_tokens": 1226.2, "avg_output_tokens": 1901.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 71.5, "median_query_seconds": 42.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 84.953, "latency": 26.258, "stderr": 1.25, "cost_per_test": 0.002168, "avg_input_tokens": 1223.4, "avg_output_tokens": 1429.1, "avg_cache_read_tokens": 16.0, "avg_cache_write_tokens": 1207.4, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 26.3, "median_query_seconds": 13.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-3": { "accuracy": 84.834, "latency": 8.42, "stderr": 1.235, "cost_per_test": 0.013966, "avg_input_tokens": 1205.4, "avg_output_tokens": 690.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 8.4, "median_query_seconds": 7.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 84.716, "latency": 8.223, "stderr": 1.239, "cost_per_test": 0.000285, "avg_input_tokens": 1271.9, "avg_output_tokens": 1543.4, "avg_reasoning_tokens": 1149.2, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 8.2, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 84.597, "latency": 2.854, "stderr": 1.243, "cost_per_test": 0.001048, "avg_input_tokens": 1217.0, "avg_output_tokens": 350.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 2.9, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 84.479, "latency": 2.493, "stderr": 1.246, "cost_per_test": 0.000428, "avg_input_tokens": 1271.9, "avg_output_tokens": 752.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 2.5, "median_query_seconds": 2.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 84.479, "latency": 12.961, "stderr": 1.246, "cost_per_test": 0.001274, "avg_input_tokens": 1217.3, "avg_output_tokens": 198.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 13.0, "median_query_seconds": 11.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 84.242, "latency": 4.536, "stderr": 1.254, "cost_per_test": 0.001322, "avg_input_tokens": 1316.4, "avg_output_tokens": 2117.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 4.5, "median_query_seconds": 3.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 84.242, "latency": 42.971, "stderr": 1.266, "cost_per_test": 0.001145, "avg_input_tokens": 1284.0, "avg_output_tokens": 1586.5, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 43.0, "median_query_seconds": 34.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 84.123, "latency": 5.825, "stderr": 1.258, "cost_per_test": 0.007887, "avg_input_tokens": 1331.1, "avg_output_tokens": 259.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 5.8, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 83.649, "latency": 2.017, "stderr": 1.273, "cost_per_test": 0.000231, "avg_input_tokens": 1271.9, "avg_output_tokens": 259.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 2.0, "median_query_seconds": 1.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 82.583, "latency": 1.975, "stderr": 1.305, "cost_per_test": 0.000171, "avg_input_tokens": 1367.4, "avg_output_tokens": 127.3, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1104.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 2.0, "median_query_seconds": 1.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 81.872, "latency": 34.574, "stderr": 1.677, "cost_per_test": 0.000387, "avg_input_tokens": 1300.1, "avg_output_tokens": 1561.3, "avg_reasoning_tokens": 1311.5, "avg_cache_read_tokens": 1110.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 34.6, "median_query_seconds": 23.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 81.517, "latency": 17.711, "stderr": 1.336, "cost_per_test": 0.001325, "avg_input_tokens": 1216.0, "avg_output_tokens": 3161.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 17.7, "median_query_seconds": 14.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 81.398, "latency": 43.071, "stderr": 1.343, "cost_per_test": 0.012351, "avg_input_tokens": 1253.4, "avg_output_tokens": 1977.7, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 43.1, "median_query_seconds": 33.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 81.28, "latency": 7.483, "stderr": 1.407, "cost_per_test": 0.00664, "avg_input_tokens": 1217.0, "avg_output_tokens": 359.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 7.5, "median_query_seconds": 7.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 81.161, "latency": 6.062, "stderr": 1.413, "cost_per_test": 0.000222, "avg_input_tokens": 1216.2, "avg_output_tokens": 404.1, "avg_reasoning_tokens": 285.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 6.1, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 81.161, "latency": 6.655, "stderr": 1.374, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-2-1212": { "accuracy": 80.924, "latency": 5.024, "stderr": 1.352, "cost_per_test": 0.004702, "avg_input_tokens": 1237.9, "avg_output_tokens": 222.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 5.0, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 80.924, "latency": 34.704, "stderr": 1.371, "cost_per_test": null, "avg_input_tokens": 1250.0, "avg_output_tokens": 1128.9, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 34.7, "median_query_seconds": 29.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 80.806, "latency": 4.392, "stderr": 1.356, "cost_per_test": 0.001227, "avg_input_tokens": 1217.3, "avg_output_tokens": 146.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 4.4, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 80.687, "latency": 2.351, "stderr": 1.359, "cost_per_test": 0.002155, "avg_input_tokens": 1271.9, "avg_output_tokens": 113.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 2.4, "median_query_seconds": 2.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 80.687, "latency": 88.62, "stderr": 1.407, "cost_per_test": 0.001115, "avg_input_tokens": 1263.1, "avg_output_tokens": 2426.9, "avg_reasoning_tokens": 2183.4, "avg_cache_read_tokens": 1087.3, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 88.6, "median_query_seconds": 46.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 79.858, "latency": 3.751, "stderr": 1.381, "cost_per_test": 0.001456, "avg_input_tokens": 1254.0, "avg_output_tokens": 400.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 3.8, "median_query_seconds": 3.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 79.858, "latency": 6.954, "stderr": 1.381, "cost_per_test": 0.006085, "avg_input_tokens": 1217.0, "avg_output_tokens": 304.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 7.0, "median_query_seconds": 6.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 79.858, "latency": 15.644, "stderr": 1.531, "cost_per_test": null, "avg_input_tokens": 1247.0, "avg_output_tokens": 703.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 15.6, "median_query_seconds": 15.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 78.91, "latency": 4.802, "stderr": 1.404, "cost_per_test": 0.000468, "avg_input_tokens": 1209.1, "avg_output_tokens": 424.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 4.8, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 78.436, "latency": 14.323, "stderr": 1.535, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 78.318, "latency": 2.074, "stderr": 1.418, "cost_per_test": 0.001013, "avg_input_tokens": 1326.4, "avg_output_tokens": 1495.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 2.1, "median_query_seconds": 1.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 78.199, "latency": 60.82, "stderr": 1.421, "cost_per_test": 0.032339, "avg_input_tokens": 1266.0, "avg_output_tokens": 4058.6, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 785, "avg_turns": 1.0, "avg_query_seconds": 60.8, "median_query_seconds": 57.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 77.725, "latency": 5.885, "stderr": 1.535, "cost_per_test": 0.006694, "avg_input_tokens": 1271.1, "avg_output_tokens": 351.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 5.9, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 76.066, "latency": 5.326, "stderr": 1.469, "cost_per_test": 0.004166, "avg_input_tokens": 1350.0, "avg_output_tokens": 244.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 5.3, "median_query_seconds": 4.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 76.066, "latency": 11.336, "stderr": 1.469, "cost_per_test": 0.00282, "avg_input_tokens": 1253.6, "avg_output_tokens": 1462.4, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 11.3, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 73.934, "latency": 1.376, "stderr": 1.513, "cost_per_test": 0.000149, "avg_input_tokens": 1271.9, "avg_output_tokens": 178.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 1.4, "median_query_seconds": 1.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 73.697, "latency": 5.701, "stderr": 1.516, "cost_per_test": 0.002098, "avg_input_tokens": 1331.1, "avg_output_tokens": 258.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 5.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 73.578, "latency": 2.73, "stderr": 1.518, "cost_per_test": 0.000167, "avg_input_tokens": 1253.6, "avg_output_tokens": 243.3, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 2.7, "median_query_seconds": 2.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 73.341, "latency": 4.476, "stderr": 1.522, "cost_per_test": 0.000352, "avg_input_tokens": 1217.0, "avg_output_tokens": 283.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 4.5, "median_query_seconds": 3.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 73.223, "latency": 2.07, "stderr": 1.562, "cost_per_test": 0.00028, "avg_input_tokens": 1217.0, "avg_output_tokens": 394.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 2.1, "median_query_seconds": 1.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 71.327, "latency": 3.519, "stderr": 1.557, "cost_per_test": 0.000429, "avg_input_tokens": 1253.6, "avg_output_tokens": 297.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 3.5, "median_query_seconds": 3.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 55.45, "latency": 5.624, "stderr": 1.713, "cost_per_test": 0.004016, "avg_input_tokens": 1317.1, "avg_output_tokens": 172.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 5.6, "median_query_seconds": 4.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "cohere/command-r-plus": { "accuracy": 52.251, "latency": 4.52, "stderr": 1.72, "cost_per_test": 0.005444, "avg_input_tokens": 1271.1, "avg_output_tokens": 226.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 4.5, "median_query_seconds": 3.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 29.621, "latency": 1.406, "stderr": 1.57, "cost_per_test": 0.00032, "avg_input_tokens": 1317.1, "avg_output_tokens": 141.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 844, "avg_turns": 1.0, "avg_query_seconds": 1.4, "median_query_seconds": 1.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "engineering": { "anthropic/claude-fable-5-1": { "accuracy": 90.815, "latency": 66.832, "stderr": 0.928, "cost_per_test": 0.303892, "avg_input_tokens": 1645.3, "avg_output_tokens": 5666.7, "avg_cache_write_tokens": 1641.3, "turn_metrics": { "tasks_measured": 968, "avg_turns": 1.0, "avg_query_seconds": 66.8, "median_query_seconds": 12.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 89.99, "latency": 80.54, "stderr": 0.964, "cost_per_test": 0.189415, "avg_input_tokens": 1643.1, "avg_output_tokens": 3453.7, "avg_cache_write_tokens": 119.4, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 80.5, "median_query_seconds": 21.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 89.886, "latency": 24.005, "stderr": 0.99, "cost_per_test": 0.059264, "avg_input_tokens": 1643.1, "avg_output_tokens": 1959.9, "avg_reasoning_tokens": 1552.4, "avg_cache_write_tokens": 1641.1, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 24.0, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 89.267, "latency": 7.272, "stderr": 1.003, "cost_per_test": 0.023229, "avg_input_tokens": 1351.9, "avg_output_tokens": 2826.8, "avg_reasoning_tokens": 2348.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 7.3, "median_query_seconds": 4.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 88.751, "latency": 14.793, "stderr": 1.019, "cost_per_test": 0.048753, "avg_input_tokens": 1351.9, "avg_output_tokens": 6230.0, "avg_reasoning_tokens": 5713.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 14.8, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 88.132, "latency": 49.792, "stderr": 1.043, "cost_per_test": 0.024833, "avg_input_tokens": 1405.1, "avg_output_tokens": 5711.6, "avg_reasoning_tokens": 5319.5, "avg_cache_read_tokens": 1065.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 968, "avg_turns": 1.0, "avg_query_seconds": 49.8, "median_query_seconds": 23.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 88.029, "latency": 64.579, "stderr": 1.043, "cost_per_test": 0.079443, "avg_input_tokens": 1351.9, "avg_output_tokens": 6394.9, "avg_reasoning_tokens": 5741.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 64.6, "median_query_seconds": 32.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 88.029, "latency": 319.67, "stderr": 1.091, "cost_per_test": 0.248968, "avg_input_tokens": 1647.5, "avg_output_tokens": 9625.3, "avg_cache_read_tokens": 8.2, "avg_cache_write_tokens": 109.1, "turn_metrics": { "tasks_measured": 968, "avg_turns": 1.0, "avg_query_seconds": 319.7, "median_query_seconds": 215.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 87.822, "latency": 30.126, "stderr": 1.051, "cost_per_test": 0.061489, "avg_input_tokens": 1351.9, "avg_output_tokens": 4898.7, "avg_reasoning_tokens": 4300.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 30.1, "median_query_seconds": 16.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 87.822, "latency": 46.83, "stderr": 1.051, "cost_per_test": 0.096053, "avg_input_tokens": 1268.5, "avg_output_tokens": 2990.4, "avg_reasoning_tokens": 2788.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 46.8, "median_query_seconds": 13.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 87.719, "latency": 72.117, "stderr": 1.054, "cost_per_test": 0.177437, "avg_input_tokens": 1643.1, "avg_output_tokens": 6762.9, "avg_cache_write_tokens": 119.4, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 72.1, "median_query_seconds": 18.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 87.616, "latency": 52.3, "stderr": 1.058, "cost_per_test": 0.080638, "avg_input_tokens": 1351.9, "avg_output_tokens": 6494.5, "avg_reasoning_tokens": 5715.5, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 52.3, "median_query_seconds": 43.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 87.307, "latency": 66.298, "stderr": 1.069, "cost_per_test": 0.037611, "avg_input_tokens": 1351.9, "avg_output_tokens": 12311.8, "avg_reasoning_tokens": 11682.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 66.3, "median_query_seconds": 62.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 87.203, "latency": 454.689, "stderr": 1.105, "cost_per_test": 0.040013, "avg_input_tokens": 1295.3, "avg_output_tokens": 6491.3, "avg_reasoning_tokens": 6150.1, "avg_cache_read_tokens": 1016.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 454.7, "median_query_seconds": 390.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 87.1, "latency": 225.755, "stderr": 1.159, "cost_per_test": 0.236159, "avg_input_tokens": 1442.3, "avg_output_tokens": 15455.4, "turn_metrics": { "tasks_measured": 966, "avg_turns": 1.0, "avg_query_seconds": 225.8, "median_query_seconds": 40.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 87.1, "latency": 463.727, "stderr": 1.175, "cost_per_test": 0.053424, "avg_input_tokens": 1261.1, "avg_output_tokens": 13460.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 764.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 463.7, "median_query_seconds": 233.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 86.894, "latency": 130.219, "stderr": 1.084, "cost_per_test": 0.072667, "avg_input_tokens": 1340.8, "avg_output_tokens": 4793.5, "avg_reasoning_tokens": 4547.7, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 130.2, "median_query_seconds": 38.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 86.687, "latency": 15.24, "stderr": 1.091, "cost_per_test": 0.029075, "avg_input_tokens": 1351.9, "avg_output_tokens": 3606.2, "avg_reasoning_tokens": 3125.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 15.2, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 86.687, "latency": 186.045, "stderr": 1.091, "cost_per_test": 0.054036, "avg_input_tokens": 1394.1, "avg_output_tokens": 8839.7, "avg_reasoning_tokens": 8432.9, "avg_cache_read_tokens": 1022.9, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 186.0, "median_query_seconds": 57.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 86.687, "latency": 877.3, "stderr": 1.139, "cost_per_test": 0.099767, "avg_input_tokens": 1330.0, "avg_output_tokens": 25086.7, "avg_reasoning_tokens": 24767.4, "avg_cache_read_tokens": 1044.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 877.3, "median_query_seconds": 627.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 86.481, "latency": 97.048, "stderr": 1.546, "cost_per_test": 0.575076, "avg_input_tokens": 1455.2, "avg_output_tokens": 7376.7, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 97.0, "median_query_seconds": 77.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 86.275, "latency": 105.172, "stderr": 1.105, "cost_per_test": 0.029168, "avg_input_tokens": 1248.1, "avg_output_tokens": 6748.4, "avg_reasoning_tokens": 6314.0, "avg_cache_read_tokens": 974.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 965, "avg_turns": 1.0, "avg_query_seconds": 105.2, "median_query_seconds": 45.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 85.759, "latency": 61.801, "stderr": 1.123, "cost_per_test": 0.035533, "avg_input_tokens": 1356.1, "avg_output_tokens": 4285.7, "avg_reasoning_tokens": 3624.7, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 61.8, "median_query_seconds": 47.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 85.759, "latency": 151.687, "stderr": 1.123, "cost_per_test": 0.038794, "avg_input_tokens": 1295.3, "avg_output_tokens": 6272.4, "avg_reasoning_tokens": 6029.6, "avg_cache_read_tokens": 953.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 151.7, "median_query_seconds": 43.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 85.759, "latency": 206.532, "stderr": 1.123, "cost_per_test": 0.017356, "avg_input_tokens": 1270.4, "avg_output_tokens": 14353.4, "avg_cache_read_tokens": 1036.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 206.5, "median_query_seconds": 77.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 85.655, "latency": 53.431, "stderr": 1.187, "cost_per_test": 0.009616, "avg_input_tokens": 1262.4, "avg_output_tokens": 7802.9, "avg_reasoning_tokens": 7576.3, "avg_cache_read_tokens": 1.2, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 53.4, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 85.655, "latency": 206.526, "stderr": 1.126, "cost_per_test": 0.126553, "avg_input_tokens": 1270.4, "avg_output_tokens": 13332.8, "avg_cache_read_tokens": 1000.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 206.5, "median_query_seconds": 66.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 85.243, "latency": 55.333, "stderr": 1.381, "cost_per_test": 0.123866, "avg_input_tokens": 1291.3, "avg_output_tokens": 4190.8, "avg_reasoning_tokens": 3851.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 55.3, "median_query_seconds": 24.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 85.243, "latency": 152.966, "stderr": 1.139, "cost_per_test": 0.015775, "avg_input_tokens": 1356.2, "avg_output_tokens": 6346.7, "avg_reasoning_tokens": 5598.7, "turn_metrics": { "tasks_measured": 968, "avg_turns": 1.0, "avg_query_seconds": 153.0, "median_query_seconds": 144.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 85.243, "latency": 254.237, "stderr": 1.491, "cost_per_test": 0.309322, "avg_input_tokens": 1443.2, "avg_output_tokens": 20331.3, "avg_cache_write_tokens": 29.9, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 254.2, "median_query_seconds": 63.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 84.83, "latency": 79.599, "stderr": 1.152, "cost_per_test": 0.062984, "avg_input_tokens": 1343.4, "avg_output_tokens": 3275.2, "avg_reasoning_tokens": 3006.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 79.6, "median_query_seconds": 44.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 84.623, "latency": 79.232, "stderr": 1.159, "cost_per_test": 0.003009, "avg_input_tokens": 1303.1, "avg_output_tokens": 4694.3, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 79.2, "median_query_seconds": 49.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 84.314, "latency": 82.131, "stderr": 1.168, "cost_per_test": 0.016678, "avg_input_tokens": 1356.1, "avg_output_tokens": 5333.3, "avg_reasoning_tokens": 4823.6, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 82.1, "median_query_seconds": 74.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.3": { "accuracy": 84.211, "latency": 426.7, "stderr": 1.171, "cost_per_test": 0.024835, "avg_input_tokens": 1255.3, "avg_output_tokens": 9704.0, "avg_reasoning_tokens": 9347.6, "avg_cache_read_tokens": 946.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 426.7, "median_query_seconds": 402.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.1": { "accuracy": 84.029, "latency": 266.126, "stderr": 1.184, "cost_per_test": 0.049265, "avg_input_tokens": 1261.3, "avg_output_tokens": 15237.6, "avg_reasoning_tokens": 14860.0, "avg_cache_read_tokens": 946.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 958, "avg_turns": 1.0, "avg_query_seconds": 266.1, "median_query_seconds": 142.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3": { "accuracy": 84.004, "latency": 175.352, "stderr": 1.178, "cost_per_test": 0.047363, "avg_input_tokens": 1268.8, "avg_output_tokens": 10465.7, "avg_reasoning_tokens": 10150.9, "avg_cache_read_tokens": 405.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 175.4, "median_query_seconds": 29.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 83.798, "latency": 89.832, "stderr": 1.187, "cost_per_test": 0.018821, "avg_input_tokens": 1262.9, "avg_output_tokens": 6159.6, "avg_cache_read_tokens": 753.7, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 89.8, "median_query_seconds": 67.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 83.695, "latency": 14.796, "stderr": 1.187, "cost_per_test": 0.001948, "avg_input_tokens": 1351.9, "avg_output_tokens": 4532.1, "avg_reasoning_tokens": 3950.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 14.8, "median_query_seconds": 10.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.2": { "accuracy": 83.385, "latency": 129.896, "stderr": 1.196, "cost_per_test": 0.037616, "avg_input_tokens": 1268.8, "avg_output_tokens": 8300.1, "avg_reasoning_tokens": 7672.2, "avg_cache_read_tokens": 597.4, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 129.9, "median_query_seconds": 56.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 83.385, "latency": 164.764, "stderr": 1.216, "cost_per_test": 0.042054, "avg_input_tokens": 1343.0, "avg_output_tokens": 10505.1, "avg_reasoning_tokens": 10271.0, "avg_cache_read_tokens": 1033.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 164.8, "median_query_seconds": 45.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 83.282, "latency": 222.254, "stderr": 1.199, "cost_per_test": 0.003153, "avg_input_tokens": 1250.5, "avg_output_tokens": 7458.4, "avg_reasoning_tokens": 7271.6, "avg_cache_read_tokens": 1191.6, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 222.3, "median_query_seconds": 132.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 83.179, "latency": 32.25, "stderr": 1.202, "cost_per_test": 0.032227, "avg_input_tokens": 1369.6, "avg_output_tokens": 5228.4, "avg_reasoning_tokens": 4898.1, "avg_cache_read_tokens": 1045.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 32.2, "median_query_seconds": 13.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 83.179, "latency": 52.692, "stderr": 1.202, "cost_per_test": 0.009277, "avg_input_tokens": 1330.0, "avg_output_tokens": 6915.7, "avg_reasoning_tokens": 6705.0, "avg_cache_read_tokens": 1026.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 52.7, "median_query_seconds": 13.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5-thinking": { "accuracy": 83.179, "latency": 198.893, "stderr": 1.205, "cost_per_test": 0.030891, "avg_input_tokens": 1261.8, "avg_output_tokens": 9528.7, "avg_cache_read_tokens": 918.2, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 198.9, "median_query_seconds": 127.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 82.972, "latency": 116.232, "stderr": 1.21, "cost_per_test": 0.1194, "avg_input_tokens": 1334.4, "avg_output_tokens": 16443.1, "avg_reasoning_tokens": 8079.9, "avg_cache_read_tokens": 12.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 116.2, "median_query_seconds": 34.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 82.869, "latency": 26.937, "stderr": 1.21, "cost_per_test": 0.02277, "avg_input_tokens": 1262.4, "avg_output_tokens": 1687.1, "avg_reasoning_tokens": 1488.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 26.9, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 82.766, "latency": 9.671, "stderr": 1.213, "cost_per_test": 0.009929, "avg_input_tokens": 1351.9, "avg_output_tokens": 3809.4, "avg_reasoning_tokens": 3319.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 9.7, "median_query_seconds": 7.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 82.766, "latency": 132.631, "stderr": 1.213, "cost_per_test": null, "avg_input_tokens": 1363.4, "avg_output_tokens": 12927.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 132.6, "median_query_seconds": 38.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 82.456, "latency": 56.165, "stderr": 1.222, "cost_per_test": 0.002943, "avg_input_tokens": 1356.1, "avg_output_tokens": 7018.2, "avg_reasoning_tokens": 6227.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 56.2, "median_query_seconds": 50.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 82.456, "latency": 81.642, "stderr": 1.606, "cost_per_test": 0.093312, "avg_input_tokens": 1454.2, "avg_output_tokens": 5930.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 81.6, "median_query_seconds": 85.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-0709": { "accuracy": 82.456, "latency": 257.063, "stderr": 1.222, "cost_per_test": 0.147419, "avg_input_tokens": 1249.3, "avg_output_tokens": 9578.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 257.1, "median_query_seconds": 167.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 82.353, "latency": 79.758, "stderr": 1.225, "cost_per_test": 0.035999, "avg_input_tokens": 1398.1, "avg_output_tokens": 11766.8, "avg_reasoning_tokens": 11368.5, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 79.8, "median_query_seconds": 28.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 82.043, "latency": 51.562, "stderr": 1.239, "cost_per_test": 0.010098, "avg_input_tokens": 1262.4, "avg_output_tokens": 4891.4, "avg_reasoning_tokens": 4610.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 51.6, "median_query_seconds": 20.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 82.043, "latency": 60.787, "stderr": 1.268, "cost_per_test": 0.265746, "avg_input_tokens": 1454.2, "avg_output_tokens": 3252.4, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 60.8, "median_query_seconds": 46.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 81.837, "latency": 105.718, "stderr": 1.239, "cost_per_test": 0.008321, "avg_input_tokens": 1284.7, "avg_output_tokens": 6542.9, "avg_cache_read_tokens": 34.7, "avg_cache_write_tokens": 1248.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 105.7, "median_query_seconds": 72.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 81.631, "latency": 19.936, "stderr": 1.474, "cost_per_test": 0.037502, "avg_input_tokens": 1643.1, "avg_output_tokens": 2089.5, "avg_cache_write_tokens": 1641.1, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 19.9, "median_query_seconds": 8.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 81.631, "latency": 196.997, "stderr": 1.244, "cost_per_test": 0.004642, "avg_input_tokens": 1268.8, "avg_output_tokens": 18333.6, "avg_cache_read_tokens": 611.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 197.0, "median_query_seconds": 18.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 81.631, "latency": 238.952, "stderr": 1.244, "cost_per_test": 0.065445, "avg_input_tokens": 1346.0, "avg_output_tokens": 10638.2, "avg_reasoning_tokens": 9622.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 239.0, "median_query_seconds": 175.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 81.115, "latency": 136.739, "stderr": 1.29, "cost_per_test": 0.017722, "avg_input_tokens": 1422.7, "avg_output_tokens": 7252.8, "avg_cache_read_tokens": 1121.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 136.7, "median_query_seconds": 43.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 81.115, "latency": 217.255, "stderr": 1.375, "cost_per_test": 0.00665, "avg_input_tokens": 1587.1, "avg_output_tokens": 7478.8, "avg_reasoning_tokens": 7174.5, "avg_cache_read_tokens": 1251.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 217.3, "median_query_seconds": 130.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 81.011, "latency": 77.878, "stderr": 1.263, "cost_per_test": 0.00212, "avg_input_tokens": 1583.0, "avg_output_tokens": 7399.9, "avg_reasoning_tokens": 7101.8, "avg_cache_read_tokens": 1266.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 77.9, "median_query_seconds": 51.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 80.599, "latency": 98.624, "stderr": 1.27, "cost_per_test": 0.062582, "avg_input_tokens": 1260.5, "avg_output_tokens": 6100.9, "avg_reasoning_tokens": 5876.9, "avg_cache_read_tokens": 2.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 954, "avg_turns": 1.0, "avg_query_seconds": 98.6, "median_query_seconds": 76.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 80.495, "latency": 60.928, "stderr": 1.278, "cost_per_test": 0.058109, "avg_input_tokens": 1262.4, "avg_output_tokens": 5653.1, "avg_reasoning_tokens": 5159.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 60.9, "median_query_seconds": 37.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 80.289, "latency": 0.0, "stderr": 1.278, "cost_per_test": 0.014173, "avg_input_tokens": 1344.1, "avg_output_tokens": 2093.3, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 80.289, "latency": 63.374, "stderr": 1.52, "cost_per_test": 0.027306, "avg_input_tokens": 1250.0, "avg_output_tokens": 4744.2, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 63.4, "median_query_seconds": 47.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 80.083, "latency": 22.173, "stderr": 1.283, "cost_per_test": 0.00358, "avg_input_tokens": 1359.3, "avg_output_tokens": 6616.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 22.2, "median_query_seconds": 10.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 79.773, "latency": 38.297, "stderr": 1.29, "cost_per_test": 0.013937, "avg_input_tokens": 1348.9, "avg_output_tokens": 1225.1, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 38.3, "median_query_seconds": 29.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 79.773, "latency": 71.805, "stderr": 1.29, "cost_per_test": 0.003066, "avg_input_tokens": 1398.3, "avg_output_tokens": 5895.6, "avg_reasoning_tokens": 5671.5, "avg_cache_read_tokens": 1079.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 71.8, "median_query_seconds": 34.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 79.154, "latency": 110.064, "stderr": 1.31, "cost_per_test": 0.005318, "avg_input_tokens": 1344.0, "avg_output_tokens": 5707.7, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 110.1, "median_query_seconds": 85.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 78.638, "latency": 47.699, "stderr": 1.317, "cost_per_test": 0.029288, "avg_input_tokens": 1262.4, "avg_output_tokens": 3345.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 47.7, "median_query_seconds": 26.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.7": { "accuracy": 78.638, "latency": 291.489, "stderr": 1.317, "cost_per_test": 0.020993, "avg_input_tokens": 1262.1, "avg_output_tokens": 9368.9, "avg_cache_read_tokens": 678.1, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 291.5, "median_query_seconds": 192.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 78.225, "latency": 81.498, "stderr": 1.326, "cost_per_test": 0.086574, "avg_input_tokens": 1362.4, "avg_output_tokens": 11270.7, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 81.5, "median_query_seconds": 75.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 77.709, "latency": 13.94, "stderr": 1.354, "cost_per_test": 0.073918, "avg_input_tokens": 1426.2, "avg_output_tokens": 629.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 1423.2, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 13.9, "median_query_seconds": 13.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 77.606, "latency": 47.81, "stderr": 1.346, "cost_per_test": 0.000647, "avg_input_tokens": 1249.9, "avg_output_tokens": 1286.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 961.8, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 47.8, "median_query_seconds": 42.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 77.503, "latency": 106.872, "stderr": 1.356, "cost_per_test": 0.004868, "avg_input_tokens": 1364.6, "avg_output_tokens": 21925.4, "avg_cache_read_tokens": 973.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 106.9, "median_query_seconds": 22.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 77.296, "latency": 98.218, "stderr": 1.346, "cost_per_test": 0.101515, "avg_input_tokens": 1454.2, "avg_output_tokens": 6476.8, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 98.2, "median_query_seconds": 103.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 77.193, "latency": 20.916, "stderr": 1.348, "cost_per_test": 0.002718, "avg_input_tokens": 1348.9, "avg_output_tokens": 4819.5, "avg_reasoning_tokens": 3894.1, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 20.9, "median_query_seconds": 16.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-4.6": { "accuracy": 77.193, "latency": 105.568, "stderr": 1.354, "cost_per_test": 0.015562, "avg_input_tokens": 1263.2, "avg_output_tokens": 6765.6, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 105.6, "median_query_seconds": 52.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 76.987, "latency": 104.441, "stderr": 1.352, "cost_per_test": 0.013442, "avg_input_tokens": 1344.1, "avg_output_tokens": 1971.6, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 968, "avg_turns": 1.0, "avg_query_seconds": 104.4, "median_query_seconds": 46.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 76.471, "latency": 16.513, "stderr": 1.363, "cost_per_test": 0.00273, "avg_input_tokens": 1348.9, "avg_output_tokens": 4710.7, "avg_reasoning_tokens": 3780.7, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 16.5, "median_query_seconds": 12.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 76.367, "latency": 14.014, "stderr": 1.365, "cost_per_test": 0.067022, "avg_input_tokens": 1426.2, "avg_output_tokens": 608.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 14.0, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 76.367, "latency": 47.029, "stderr": 1.365, "cost_per_test": 0.004227, "avg_input_tokens": 1249.3, "avg_output_tokens": 7704.8, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 47.0, "median_query_seconds": 33.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 76.161, "latency": 11.153, "stderr": 1.543, "cost_per_test": 0.072202, "avg_input_tokens": 1426.2, "avg_output_tokens": 677.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 11.2, "median_query_seconds": 10.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 75.439, "latency": 142.331, "stderr": 1.392, "cost_per_test": 0.015702, "avg_input_tokens": 1287.7, "avg_output_tokens": 6346.6, "avg_cache_read_tokens": 36.0, "avg_cache_write_tokens": 1251.7, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 142.3, "median_query_seconds": 78.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 74.923, "latency": 26.843, "stderr": 1.595, "cost_per_test": 0.002489, "avg_input_tokens": 1350.3, "avg_output_tokens": 1216.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 26.8, "median_query_seconds": 21.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 74.613, "latency": 89.219, "stderr": 1.398, "cost_per_test": 0.013803, "avg_input_tokens": 1261.4, "avg_output_tokens": 6744.0, "avg_reasoning_tokens": 6420.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 927, "avg_turns": 1.0, "avg_query_seconds": 89.2, "median_query_seconds": 57.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 74.613, "latency": 146.95, "stderr": 1.415, "cost_per_test": 0.012338, "avg_input_tokens": 1273.2, "avg_output_tokens": 9890.2, "avg_cache_read_tokens": 22.6, "avg_cache_write_tokens": 1250.5, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 146.9, "median_query_seconds": 61.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 74.2, "latency": 62.165, "stderr": 1.406, "cost_per_test": 0.232772, "avg_input_tokens": 1262.4, "avg_output_tokens": 3563.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 62.2, "median_query_seconds": 48.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 73.787, "latency": 67.263, "stderr": 1.413, "cost_per_test": 0.087638, "avg_input_tokens": 1454.2, "avg_output_tokens": 5551.7, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 67.3, "median_query_seconds": 70.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 73.271, "latency": 26.91, "stderr": 1.465, "cost_per_test": 0.001399, "avg_input_tokens": 1284.8, "avg_output_tokens": 3337.8, "avg_reasoning_tokens": 2935.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 26.9, "median_query_seconds": 12.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 72.446, "latency": 67.828, "stderr": 1.465, "cost_per_test": 0.001946, "avg_input_tokens": 1250.0, "avg_output_tokens": 912.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 67.8, "median_query_seconds": 56.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 72.136, "latency": 24.109, "stderr": 1.442, "cost_per_test": 0.010718, "avg_input_tokens": 1262.4, "avg_output_tokens": 2120.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 24.1, "median_query_seconds": 17.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 71.93, "latency": 13.897, "stderr": 1.443, "cost_per_test": 0.001665, "avg_input_tokens": 1249.3, "avg_output_tokens": 2581.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 13.9, "median_query_seconds": 12.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 71.723, "latency": 59.966, "stderr": 1.57, "cost_per_test": 0.044915, "avg_input_tokens": 1454.2, "avg_output_tokens": 8692.1, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 60.0, "median_query_seconds": 51.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 71.517, "latency": 49.348, "stderr": 1.455, "cost_per_test": 0.004726, "avg_input_tokens": 1329.1, "avg_output_tokens": 7584.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 49.3, "median_query_seconds": 33.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 71.414, "latency": 20.77, "stderr": 1.451, "cost_per_test": 0.00048, "avg_input_tokens": 1348.9, "avg_output_tokens": 5927.6, "avg_reasoning_tokens": 5065.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 20.8, "median_query_seconds": 16.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 70.898, "latency": 7.294, "stderr": 1.459, "cost_per_test": 0.000875, "avg_input_tokens": 1251.1, "avg_output_tokens": 681.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 7.3, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 70.898, "latency": 12.865, "stderr": 1.459, "cost_per_test": 0.013775, "avg_input_tokens": 1426.2, "avg_output_tokens": 633.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 12.9, "median_query_seconds": 11.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 69.453, "latency": 44.845, "stderr": 1.48, "cost_per_test": 0.004529, "avg_input_tokens": 1271.8, "avg_output_tokens": 1085.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 44.8, "median_query_seconds": 29.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 69.35, "latency": 169.443, "stderr": 1.516, "cost_per_test": 0.003996, "avg_input_tokens": 1261.6, "avg_output_tokens": 6873.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 169.4, "median_query_seconds": 102.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-3": { "accuracy": 69.143, "latency": 16.967, "stderr": 1.484, "cost_per_test": 0.023894, "avg_input_tokens": 1248.3, "avg_output_tokens": 1343.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 17.0, "median_query_seconds": 14.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 68.627, "latency": 50.797, "stderr": 1.491, "cost_per_test": 0.029435, "avg_input_tokens": 1262.4, "avg_output_tokens": 6374.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 50.8, "median_query_seconds": 35.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 68.524, "latency": 6.686, "stderr": 1.492, "cost_per_test": 0.001175, "avg_input_tokens": 1348.9, "avg_output_tokens": 2601.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 6.7, "median_query_seconds": 5.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 67.802, "latency": 16.506, "stderr": 1.501, "cost_per_test": 0.011514, "avg_input_tokens": 1263.4, "avg_output_tokens": 1123.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 16.5, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 67.699, "latency": 9.302, "stderr": 1.502, "cost_per_test": 0.000771, "avg_input_tokens": 1348.9, "avg_output_tokens": 1589.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 9.3, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-4.5": { "accuracy": 67.389, "latency": 379.722, "stderr": 1.506, "cost_per_test": 0.021658, "avg_input_tokens": 1261.9, "avg_output_tokens": 9500.5, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 379.7, "median_query_seconds": 194.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 66.77, "latency": 8.402, "stderr": 1.513, "cost_per_test": 0.012613, "avg_input_tokens": 1426.2, "avg_output_tokens": 555.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 8.4, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 66.151, "latency": 52.506, "stderr": 1.52, "cost_per_test": 0.004676, "avg_input_tokens": 1262.4, "avg_output_tokens": 11533.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 52.5, "median_query_seconds": 39.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 65.325, "latency": 7.809, "stderr": 1.529, "cost_per_test": 0.002335, "avg_input_tokens": 1263.4, "avg_output_tokens": 1143.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 7.8, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-2-1212": { "accuracy": 62.229, "latency": 23.215, "stderr": 1.557, "cost_per_test": 0.010247, "avg_input_tokens": 1331.3, "avg_output_tokens": 758.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 23.2, "median_query_seconds": 14.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 62.126, "latency": 6.197, "stderr": 1.558, "cost_per_test": 0.000363, "avg_input_tokens": 1410.3, "avg_output_tokens": 481.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1067.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 6.2, "median_query_seconds": 3.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 61.197, "latency": 6.929, "stderr": 1.565, "cost_per_test": 0.009441, "avg_input_tokens": 1426.2, "avg_output_tokens": 344.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 6.9, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 60.991, "latency": 7.111, "stderr": 1.567, "cost_per_test": 0.000583, "avg_input_tokens": 1251.1, "avg_output_tokens": 606.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 7.1, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 60.681, "latency": 28.367, "stderr": 1.571, "cost_per_test": 0.001626, "avg_input_tokens": 1250.0, "avg_output_tokens": 556.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 28.4, "median_query_seconds": 24.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 59.752, "latency": 7.355, "stderr": 1.575, "cost_per_test": 0.004022, "avg_input_tokens": 1348.9, "avg_output_tokens": 467.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 7.4, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 59.649, "latency": 20.497, "stderr": 1.581, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 58.308, "latency": 2.258, "stderr": 1.584, "cost_per_test": 0.001118, "avg_input_tokens": 1369.3, "avg_output_tokens": 1688.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 2.3, "median_query_seconds": 1.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 56.45, "latency": 50.126, "stderr": 1.603, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 56.45, "latency": 64.176, "stderr": 1.593, "cost_per_test": 0.030606, "avg_input_tokens": 1350.4, "avg_output_tokens": 5581.1, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 64.2, "median_query_seconds": 48.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 55.521, "latency": 18.253, "stderr": 1.606, "cost_per_test": 0.011655, "avg_input_tokens": 1368.5, "avg_output_tokens": 823.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 18.3, "median_query_seconds": 13.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 55.108, "latency": 16.182, "stderr": 1.598, "cost_per_test": 0.010385, "avg_input_tokens": 1263.4, "avg_output_tokens": 722.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 16.2, "median_query_seconds": 12.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 53.87, "latency": 83.21, "stderr": 1.603, "cost_per_test": null, "avg_input_tokens": 1281.7, "avg_output_tokens": 4578.4, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 83.2, "median_query_seconds": 72.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 52.941, "latency": 122.193, "stderr": 1.596, "cost_per_test": 0.001904, "avg_input_tokens": 1386.1, "avg_output_tokens": 9098.4, "avg_reasoning_tokens": 8645.3, "avg_cache_read_tokens": 1080.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 122.2, "median_query_seconds": 102.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 52.735, "latency": 5.892, "stderr": 1.604, "cost_per_test": 0.001797, "avg_input_tokens": 1288.7, "avg_output_tokens": 753.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 5.9, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 52.735, "latency": 31.202, "stderr": 1.604, "cost_per_test": 0.00718, "avg_input_tokens": 1350.4, "avg_output_tokens": 4336.4, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 31.2, "median_query_seconds": 26.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 52.632, "latency": 15.557, "stderr": 1.606, "cost_per_test": 0.010722, "avg_input_tokens": 1263.4, "avg_output_tokens": 756.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 15.6, "median_query_seconds": 14.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 52.219, "latency": 16.566, "stderr": 1.605, "cost_per_test": 0.007721, "avg_input_tokens": 1420.4, "avg_output_tokens": 813.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 16.6, "median_query_seconds": 13.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 52.219, "latency": 23.061, "stderr": 1.606, "cost_per_test": null, "avg_input_tokens": 1281.7, "avg_output_tokens": 1204.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 23.1, "median_query_seconds": 21.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 49.432, "latency": 7.592, "stderr": 1.606, "cost_per_test": 0.000322, "avg_input_tokens": 1350.4, "avg_output_tokens": 735.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 7.6, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 49.123, "latency": 221.89, "stderr": 1.605, "cost_per_test": 0.005474, "avg_input_tokens": 1349.1, "avg_output_tokens": 26134.5, "avg_reasoning_tokens": 12857.0, "avg_cache_read_tokens": 1068.6, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 221.9, "median_query_seconds": 207.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 46.027, "latency": 9.576, "stderr": 1.601, "cost_per_test": 0.000792, "avg_input_tokens": 1350.4, "avg_output_tokens": 869.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 9.6, "median_query_seconds": 7.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 45.924, "latency": 6.225, "stderr": 1.601, "cost_per_test": 0.002363, "avg_input_tokens": 1426.2, "avg_output_tokens": 305.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 6.2, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 44.788, "latency": 2.879, "stderr": 1.597, "cost_per_test": 0.000227, "avg_input_tokens": 1348.9, "avg_output_tokens": 419.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 2.9, "median_query_seconds": 2.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 39.216, "latency": 8.948, "stderr": 1.568, "cost_per_test": 0.000561, "avg_input_tokens": 1263.4, "avg_output_tokens": 619.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 8.9, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 35.913, "latency": 5.105, "stderr": 1.527, "cost_per_test": 0.000638, "avg_input_tokens": 1263.4, "avg_output_tokens": 1279.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 5.1, "median_query_seconds": 4.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 31.579, "latency": 19.41, "stderr": 1.484, "cost_per_test": 0.007021, "avg_input_tokens": 1421.5, "avg_output_tokens": 522.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 19.4, "median_query_seconds": 17.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "cohere/command-r-plus": { "accuracy": 26.109, "latency": 7.918, "stderr": 1.411, "cost_per_test": 0.00741, "avg_input_tokens": 1368.5, "avg_output_tokens": 398.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 7.9, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 17.441, "latency": 5.321, "stderr": 1.213, "cost_per_test": 0.000545, "avg_input_tokens": 1421.5, "avg_output_tokens": 652.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 969, "avg_turns": 1.0, "avg_query_seconds": 5.3, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "health": { "anthropic/claude-opus-5": { "accuracy": 87.775, "latency": 8.814, "stderr": 1.145, "cost_per_test": 0.023061, "avg_input_tokens": 1540.4, "avg_output_tokens": 554.4, "avg_reasoning_tokens": 296.1, "avg_cache_read_tokens": 73.9, "avg_cache_write_tokens": 1464.5, "turn_metrics": { "tasks_measured": 817, "avg_turns": 1.0, "avg_query_seconds": 8.8, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 86.675, "latency": 18.621, "stderr": 1.188, "cost_per_test": 0.081746, "avg_input_tokens": 1542.5, "avg_output_tokens": 1262.0, "avg_cache_read_tokens": 51.1, "avg_cache_write_tokens": 1487.3, "turn_metrics": { "tasks_measured": 817, "avg_turns": 1.0, "avg_query_seconds": 18.6, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 86.43, "latency": 19.713, "stderr": 1.197, "cost_per_test": 0.018611, "avg_input_tokens": 1042.8, "avg_output_tokens": 1377.1, "avg_reasoning_tokens": 1150.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 19.7, "median_query_seconds": 12.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 85.086, "latency": 17.113, "stderr": 1.246, "cost_per_test": 0.040556, "avg_input_tokens": 1540.5, "avg_output_tokens": 1321.7, "avg_cache_write_tokens": 9.1, "turn_metrics": { "tasks_measured": 814, "avg_turns": 1.0, "avg_query_seconds": 17.1, "median_query_seconds": 9.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 84.963, "latency": 20.011, "stderr": 1.254, "cost_per_test": 0.085317, "avg_input_tokens": 1197.0, "avg_output_tokens": 898.2, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 20.0, "median_query_seconds": 19.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 84.719, "latency": 4.605, "stderr": 1.258, "cost_per_test": 0.008386, "avg_input_tokens": 1042.8, "avg_output_tokens": 909.6, "avg_reasoning_tokens": 744.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 4.6, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 84.597, "latency": 6.549, "stderr": 1.262, "cost_per_test": 0.010436, "avg_input_tokens": 1042.8, "avg_output_tokens": 1182.9, "avg_reasoning_tokens": 979.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 6.5, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 84.597, "latency": 20.082, "stderr": 1.262, "cost_per_test": 0.084721, "avg_input_tokens": 1540.4, "avg_output_tokens": 1323.7, "avg_cache_read_tokens": 34.7, "avg_cache_write_tokens": 1501.9, "turn_metrics": { "tasks_measured": 815, "avg_turns": 1.0, "avg_query_seconds": 20.1, "median_query_seconds": 13.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 84.597, "latency": 25.958, "stderr": 1.603, "cost_per_test": 0.024715, "avg_input_tokens": 1186.0, "avg_output_tokens": 1410.5, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 26.0, "median_query_seconds": 17.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 84.597, "latency": 29.677, "stderr": 1.446, "cost_per_test": 0.15263, "avg_input_tokens": 1198.0, "avg_output_tokens": 1795.5, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 29.7, "median_query_seconds": 19.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 84.474, "latency": 8.143, "stderr": 1.266, "cost_per_test": 0.019314, "avg_input_tokens": 1042.8, "avg_output_tokens": 2366.7, "avg_reasoning_tokens": 2177.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 8.1, "median_query_seconds": 3.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 84.352, "latency": 25.191, "stderr": 1.27, "cost_per_test": 0.013281, "avg_input_tokens": 1121.9, "avg_output_tokens": 1396.8, "avg_reasoning_tokens": 1134.3, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 25.2, "median_query_seconds": 22.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.5": { "accuracy": 84.352, "latency": 35.101, "stderr": 1.27, "cost_per_test": 0.026769, "avg_input_tokens": 1096.4, "avg_output_tokens": 1305.4, "avg_reasoning_tokens": 1238.0, "avg_cache_read_tokens": 6.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 35.1, "median_query_seconds": 21.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 84.352, "latency": 38.646, "stderr": 1.27, "cost_per_test": 0.012673, "avg_input_tokens": 1040.0, "avg_output_tokens": 1977.5, "avg_reasoning_tokens": 1831.8, "avg_cache_read_tokens": 848.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 38.6, "median_query_seconds": 17.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 84.108, "latency": 11.833, "stderr": 1.278, "cost_per_test": 0.025299, "avg_input_tokens": 1096.4, "avg_output_tokens": 660.6, "avg_reasoning_tokens": 607.8, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 11.8, "median_query_seconds": 5.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 84.108, "latency": 15.984, "stderr": 1.282, "cost_per_test": 0.015834, "avg_input_tokens": 1215.0, "avg_output_tokens": 365.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 16.0, "median_query_seconds": 15.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 83.985, "latency": 8.045, "stderr": 1.321, "cost_per_test": 0.04467, "avg_input_tokens": 1169.0, "avg_output_tokens": 361.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 8.0, "median_query_seconds": 8.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 83.985, "latency": 9.711, "stderr": 1.282, "cost_per_test": 0.041684, "avg_input_tokens": 1169.0, "avg_output_tokens": 322.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 9.7, "median_query_seconds": 9.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 83.985, "latency": 61.797, "stderr": 1.282, "cost_per_test": 0.017996, "avg_input_tokens": 1159.9, "avg_output_tokens": 2643.0, "avg_reasoning_tokens": 2536.7, "avg_cache_read_tokens": 103.9, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 61.8, "median_query_seconds": 14.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 83.985, "latency": 315.299, "stderr": 1.282, "cost_per_test": 0.011257, "avg_input_tokens": 1040.0, "avg_output_tokens": 1733.9, "avg_reasoning_tokens": 1550.4, "avg_cache_read_tokens": 817.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 315.3, "median_query_seconds": 333.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 83.741, "latency": 25.643, "stderr": 1.29, "cost_per_test": 0.001883, "avg_input_tokens": 1042.1, "avg_output_tokens": 1286.5, "avg_cache_read_tokens": 148.5, "avg_cache_write_tokens": 891.6, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 25.6, "median_query_seconds": 16.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 83.619, "latency": 10.563, "stderr": 1.294, "cost_per_test": 0.019367, "avg_input_tokens": 1042.8, "avg_output_tokens": 1440.2, "avg_reasoning_tokens": 1203.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 10.6, "median_query_seconds": 8.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 83.619, "latency": 22.817, "stderr": 1.294, "cost_per_test": 0.031299, "avg_input_tokens": 1045.2, "avg_output_tokens": 2434.0, "avg_reasoning_tokens": 2163.1, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 22.8, "median_query_seconds": 15.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 83.374, "latency": 19.769, "stderr": 1.309, "cost_per_test": 0.0099, "avg_input_tokens": 1228.4, "avg_output_tokens": 2227.4, "avg_reasoning_tokens": 2009.2, "avg_cache_read_tokens": 1001.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 19.8, "median_query_seconds": 14.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 82.763, "latency": 5.144, "stderr": 1.321, "cost_per_test": 0.006085, "avg_input_tokens": 1096.4, "avg_output_tokens": 324.3, "avg_reasoning_tokens": 277.2, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 5.1, "median_query_seconds": 3.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 82.641, "latency": 37.102, "stderr": 1.324, "cost_per_test": 0.018444, "avg_input_tokens": 1042.8, "avg_output_tokens": 5974.3, "avg_reasoning_tokens": 5685.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 37.1, "median_query_seconds": 34.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 82.641, "latency": 402.568, "stderr": 1.324, "cost_per_test": 0.00841, "avg_input_tokens": 1008.7, "avg_output_tokens": 3210.9, "avg_reasoning_tokens": 2981.8, "avg_cache_read_tokens": 836.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 402.6, "median_query_seconds": 387.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 82.396, "latency": 29.104, "stderr": 1.553, "cost_per_test": 0.03146, "avg_input_tokens": 1186.0, "avg_output_tokens": 1859.9, "avg_cache_write_tokens": 5.4, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 29.1, "median_query_seconds": 17.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 82.396, "latency": 38.712, "stderr": 1.598, "cost_per_test": 0.045974, "avg_input_tokens": 1096.4, "avg_output_tokens": 1482.0, "avg_reasoning_tokens": 1399.7, "avg_cache_read_tokens": 30.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 38.7, "median_query_seconds": 15.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 82.396, "latency": 78.119, "stderr": 1.332, "cost_per_test": 0.022124, "avg_input_tokens": 1546.2, "avg_output_tokens": 1165.3, "avg_cache_write_tokens": 7.5, "turn_metrics": { "tasks_measured": 805, "avg_turns": 1.0, "avg_query_seconds": 78.1, "median_query_seconds": 68.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 82.396, "latency": 678.193, "stderr": 1.342, "cost_per_test": 0.016856, "avg_input_tokens": 1142.3, "avg_output_tokens": 3904.2, "avg_reasoning_tokens": 3780.5, "avg_cache_read_tokens": 88.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 678.2, "median_query_seconds": 618.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 82.274, "latency": 40.906, "stderr": 1.692, "cost_per_test": 0.065913, "avg_input_tokens": 1540.4, "avg_output_tokens": 4022.2, "avg_cache_read_tokens": 56.5, "avg_cache_write_tokens": 1482.0, "turn_metrics": { "tasks_measured": 815, "avg_turns": 1.0, "avg_query_seconds": 40.9, "median_query_seconds": 12.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 82.029, "latency": 19.978, "stderr": 1.342, "cost_per_test": 0.015709, "avg_input_tokens": 1178.9, "avg_output_tokens": 1002.5, "avg_reasoning_tokens": 815.4, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 20.0, "median_query_seconds": 10.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 82.029, "latency": 29.09, "stderr": 1.49, "cost_per_test": 0.027233, "avg_input_tokens": 1197.0, "avg_output_tokens": 1576.2, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 29.1, "median_query_seconds": 19.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 82.029, "latency": 41.654, "stderr": 1.346, "cost_per_test": 0.010082, "avg_input_tokens": 1071.9, "avg_output_tokens": 2275.4, "avg_reasoning_tokens": 2006.5, "avg_cache_read_tokens": 844.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 808, "avg_turns": 1.0, "avg_query_seconds": 41.7, "median_query_seconds": 24.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 81.785, "latency": 41.946, "stderr": 1.363, "cost_per_test": 0.010657, "avg_input_tokens": 1102.9, "avg_output_tokens": 2180.5, "avg_reasoning_tokens": 1949.8, "avg_cache_read_tokens": 422.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 41.9, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-0709": { "accuracy": 81.663, "latency": 34.976, "stderr": 1.353, "cost_per_test": 0.030109, "avg_input_tokens": 1002.7, "avg_output_tokens": 1806.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 35.0, "median_query_seconds": 15.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.1": { "accuracy": 81.518, "latency": 42.893, "stderr": 1.358, "cost_per_test": 0.006832, "avg_input_tokens": 1095.9, "avg_output_tokens": 2015.0, "avg_reasoning_tokens": 1747.5, "avg_cache_read_tokens": 890.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 817, "avg_turns": 1.0, "avg_query_seconds": 42.9, "median_query_seconds": 22.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 81.418, "latency": 12.775, "stderr": 1.363, "cost_per_test": 0.001906, "avg_input_tokens": 1096.4, "avg_output_tokens": 1422.1, "avg_reasoning_tokens": 1364.1, "avg_cache_read_tokens": 111.7, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 12.8, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 81.418, "latency": 41.625, "stderr": 1.36, "cost_per_test": 0.010657, "avg_input_tokens": 1103.0, "avg_output_tokens": 2224.0, "avg_reasoning_tokens": 2024.3, "avg_cache_read_tokens": 578.6, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 41.6, "median_query_seconds": 20.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 81.296, "latency": 53.62, "stderr": 1.4, "cost_per_test": 0.010372, "avg_input_tokens": 1155.3, "avg_output_tokens": 2516.7, "avg_reasoning_tokens": 2448.4, "avg_cache_read_tokens": 876.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 53.6, "median_query_seconds": 12.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 81.174, "latency": 21.417, "stderr": 1.367, "cost_per_test": 0.022852, "avg_input_tokens": 1095.6, "avg_output_tokens": 2912.0, "avg_reasoning_tokens": 1416.3, "avg_cache_read_tokens": 3.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 21.4, "median_query_seconds": 11.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 80.929, "latency": 8.982, "stderr": 1.374, "cost_per_test": 0.010252, "avg_input_tokens": 1122.5, "avg_output_tokens": 1624.2, "avg_reasoning_tokens": 1447.3, "avg_cache_read_tokens": 965.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 9.0, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 80.929, "latency": 14.211, "stderr": 1.374, "cost_per_test": 0.011522, "avg_input_tokens": 1096.4, "avg_output_tokens": 1016.8, "avg_reasoning_tokens": 910.0, "avg_cache_read_tokens": 14.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 14.2, "median_query_seconds": 10.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 80.929, "latency": 42.236, "stderr": 1.393, "cost_per_test": 0.000983, "avg_input_tokens": 1102.9, "avg_output_tokens": 3748.4, "avg_cache_read_tokens": 618.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 42.2, "median_query_seconds": 8.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 80.807, "latency": 28.123, "stderr": 1.377, "cost_per_test": 0.017724, "avg_input_tokens": 1096.4, "avg_output_tokens": 1635.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 28.1, "median_query_seconds": 22.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 80.807, "latency": 32.16, "stderr": 1.377, "cost_per_test": 0.006303, "avg_input_tokens": 1100.9, "avg_output_tokens": 2012.6, "avg_cache_read_tokens": 790.9, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 32.2, "median_query_seconds": 22.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 80.807, "latency": 35.864, "stderr": 1.377, "cost_per_test": 0.00687, "avg_input_tokens": 1121.9, "avg_output_tokens": 2103.2, "avg_reasoning_tokens": 1917.6, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 35.9, "median_query_seconds": 33.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark": { "accuracy": 80.807, "latency": 59.9, "stderr": 1.377, "cost_per_test": 0.001615, "avg_input_tokens": 1126.4, "avg_output_tokens": 2410.3, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 59.9, "median_query_seconds": 53.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 80.685, "latency": 7.345, "stderr": 1.38, "cost_per_test": 0.008541, "avg_input_tokens": 1169.0, "avg_output_tokens": 335.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 7.3, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 80.685, "latency": 9.436, "stderr": 1.38, "cost_per_test": 0.007691, "avg_input_tokens": 1096.4, "avg_output_tokens": 687.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 9.4, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 80.685, "latency": 13.266, "stderr": 1.38, "cost_per_test": 0.001767, "avg_input_tokens": 1142.3, "avg_output_tokens": 1252.7, "avg_reasoning_tokens": 1201.8, "avg_cache_read_tokens": 913.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 13.3, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 80.685, "latency": 123.418, "stderr": 1.38, "cost_per_test": 0.013682, "avg_input_tokens": 1100.8, "avg_output_tokens": 3312.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 756.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 123.4, "median_query_seconds": 71.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 80.44, "latency": 3.872, "stderr": 1.387, "cost_per_test": 0.003157, "avg_input_tokens": 1042.8, "avg_output_tokens": 1137.8, "avg_reasoning_tokens": 951.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 3.9, "median_query_seconds": 3.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 80.318, "latency": 6.968, "stderr": 1.39, "cost_per_test": 0.000835, "avg_input_tokens": 1042.8, "avg_output_tokens": 1825.8, "avg_reasoning_tokens": 1580.7, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 7.0, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 80.318, "latency": 28.811, "stderr": 1.39, "cost_per_test": 0.026623, "avg_input_tokens": 1197.1, "avg_output_tokens": 1535.5, "turn_metrics": { "tasks_measured": 817, "avg_turns": 1.0, "avg_query_seconds": 28.8, "median_query_seconds": 20.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 80.196, "latency": 29.591, "stderr": 1.406, "cost_per_test": 0.002572, "avg_input_tokens": 1180.1, "avg_output_tokens": 978.7, "avg_cache_read_tokens": 1010.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 29.6, "median_query_seconds": 19.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5-thinking": { "accuracy": 80.196, "latency": 82.493, "stderr": 1.393, "cost_per_test": 0.008348, "avg_input_tokens": 1095.8, "avg_output_tokens": 2482.3, "avg_cache_read_tokens": 851.2, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 82.5, "median_query_seconds": 69.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 80.073, "latency": 16.235, "stderr": 1.406, "cost_per_test": 0.002732, "avg_input_tokens": 1096.4, "avg_output_tokens": 1229.6, "avg_reasoning_tokens": 1143.6, "avg_cache_read_tokens": 6.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 16.2, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 80.073, "latency": 65.695, "stderr": 1.397, "cost_per_test": 0.008763, "avg_input_tokens": 1121.9, "avg_output_tokens": 3464.1, "avg_reasoning_tokens": 3156.4, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 65.7, "median_query_seconds": 55.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 79.951, "latency": 15.379, "stderr": 1.4, "cost_per_test": 0.007625, "avg_input_tokens": 1062.0, "avg_output_tokens": 629.8, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 15.4, "median_query_seconds": 14.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 79.951, "latency": 23.197, "stderr": 1.4, "cost_per_test": 0.0985, "avg_input_tokens": 1096.4, "avg_output_tokens": 1367.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 23.2, "median_query_seconds": 17.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 79.829, "latency": 59.033, "stderr": 1.403, "cost_per_test": 0.029508, "avg_input_tokens": 1104.4, "avg_output_tokens": 3018.1, "avg_cache_read_tokens": 959.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 59.0, "median_query_seconds": 12.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 79.462, "latency": 17.419, "stderr": 1.412, "cost_per_test": null, "avg_input_tokens": 1065.6, "avg_output_tokens": 1700.6, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 17.4, "median_query_seconds": 7.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "zai/glm-4.6": { "accuracy": 79.462, "latency": 32.202, "stderr": 1.412, "cost_per_test": 0.004877, "avg_input_tokens": 1097.7, "avg_output_tokens": 1923.0, "turn_metrics": { "tasks_measured": 817, "avg_turns": 1.0, "avg_query_seconds": 32.2, "median_query_seconds": 23.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 79.34, "latency": 24.979, "stderr": 1.416, "cost_per_test": 0.028831, "avg_input_tokens": 1197.0, "avg_output_tokens": 1682.7, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 25.0, "median_query_seconds": 18.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 79.218, "latency": 15.775, "stderr": 1.419, "cost_per_test": 0.000726, "avg_input_tokens": 1151.7, "avg_output_tokens": 1274.4, "avg_reasoning_tokens": 1158.2, "avg_cache_read_tokens": 942.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 15.8, "median_query_seconds": 10.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 79.218, "latency": 35.649, "stderr": 1.748, "cost_per_test": 0.000991, "avg_input_tokens": 1349.9, "avg_output_tokens": 975.1, "avg_reasoning_tokens": 774.9, "avg_cache_read_tokens": 1027.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 35.6, "median_query_seconds": 23.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 79.218, "latency": 57.829, "stderr": 1.419, "cost_per_test": 0.003965, "avg_input_tokens": 1104.0, "avg_output_tokens": 3225.8, "avg_cache_read_tokens": 986.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 805, "avg_turns": 1.0, "avg_query_seconds": 57.8, "median_query_seconds": 14.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 78.973, "latency": 5.73, "stderr": 1.428, "cost_per_test": 0.000228, "avg_input_tokens": 1106.6, "avg_output_tokens": 886.6, "avg_cache_read_tokens": 827.6, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 4.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 78.973, "latency": 6.52, "stderr": 1.425, "cost_per_test": 0.007583, "avg_input_tokens": 1169.0, "avg_output_tokens": 271.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 6.5, "median_query_seconds": 6.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 78.851, "latency": 36.405, "stderr": 1.428, "cost_per_test": 0.00458, "avg_input_tokens": 1107.0, "avg_output_tokens": 541.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 36.4, "median_query_seconds": 29.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 78.851, "latency": 41.573, "stderr": 1.431, "cost_per_test": 0.000192, "avg_input_tokens": 1063.3, "avg_output_tokens": 319.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 953.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 41.6, "median_query_seconds": 32.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 78.729, "latency": 99.036, "stderr": 1.431, "cost_per_test": 0.022536, "avg_input_tokens": 1109.0, "avg_output_tokens": 3534.2, "avg_reasoning_tokens": 2981.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 99.0, "median_query_seconds": 77.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 78.606, "latency": 102.332, "stderr": 1.434, "cost_per_test": 0.007673, "avg_input_tokens": 1163.9, "avg_output_tokens": 2363.6, "avg_reasoning_tokens": 2247.2, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 102.3, "median_query_seconds": 36.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max": { "accuracy": 78.484, "latency": 0.0, "stderr": 1.437, "cost_per_test": 0.004609, "avg_input_tokens": 1107.0, "avg_output_tokens": 546.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 78.484, "latency": 10.916, "stderr": 1.437, "cost_per_test": 0.003669, "avg_input_tokens": 1096.4, "avg_output_tokens": 1697.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 10.9, "median_query_seconds": 9.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 78.484, "latency": 30.761, "stderr": 1.437, "cost_per_test": 0.001703, "avg_input_tokens": 1121.9, "avg_output_tokens": 3976.9, "avg_reasoning_tokens": 3702.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 30.8, "median_query_seconds": 27.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 78.362, "latency": 6.152, "stderr": 1.44, "cost_per_test": 0.008436, "avg_input_tokens": 1169.0, "avg_output_tokens": 328.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 6.2, "median_query_seconds": 6.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-3": { "accuracy": 78.362, "latency": 8.855, "stderr": 1.44, "cost_per_test": 0.013128, "avg_input_tokens": 1001.7, "avg_output_tokens": 674.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 8.9, "median_query_seconds": 8.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 78.362, "latency": 17.843, "stderr": 1.443, "cost_per_test": 0.000458, "avg_input_tokens": 1346.0, "avg_output_tokens": 1467.2, "avg_reasoning_tokens": 1345.7, "avg_cache_read_tokens": 1030.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 17.8, "median_query_seconds": 11.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 78.362, "latency": 67.657, "stderr": 1.44, "cost_per_test": 0.000779, "avg_input_tokens": 1063.4, "avg_output_tokens": 1724.6, "avg_reasoning_tokens": 1642.8, "avg_cache_read_tokens": 955.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 67.7, "median_query_seconds": 48.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-4.5": { "accuracy": 78.117, "latency": 57.607, "stderr": 1.446, "cost_per_test": 0.006468, "avg_input_tokens": 1095.9, "avg_output_tokens": 2641.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 57.6, "median_query_seconds": 48.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 77.995, "latency": 2.974, "stderr": 1.448, "cost_per_test": 0.004239, "avg_input_tokens": 1097.4, "avg_output_tokens": 255.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 3.0, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 77.751, "latency": 18.661, "stderr": 1.454, "cost_per_test": 0.009501, "avg_input_tokens": 1096.4, "avg_output_tokens": 1885.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 18.7, "median_query_seconds": 12.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 77.628, "latency": 4.928, "stderr": 1.457, "cost_per_test": 0.001017, "avg_input_tokens": 1062.0, "avg_output_tokens": 1076.1, "avg_reasoning_tokens": 796.9, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 4.9, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-4.7": { "accuracy": 77.628, "latency": 63.083, "stderr": 1.457, "cost_per_test": 0.004571, "avg_input_tokens": 1095.9, "avg_output_tokens": 1966.3, "avg_cache_read_tokens": 760.2, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 63.1, "median_query_seconds": 33.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 77.506, "latency": 15.73, "stderr": 1.46, "cost_per_test": 0.002245, "avg_input_tokens": 1109.9, "avg_output_tokens": 378.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 15.7, "median_query_seconds": 12.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 77.506, "latency": 36.157, "stderr": 1.46, "cost_per_test": 0.003324, "avg_input_tokens": 1045.1, "avg_output_tokens": 1242.2, "avg_cache_read_tokens": 141.9, "avg_cache_write_tokens": 903.2, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 36.2, "median_query_seconds": 22.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 77.384, "latency": 29.718, "stderr": 1.476, "cost_per_test": 0.002225, "avg_input_tokens": 1025.1, "avg_output_tokens": 1572.9, "avg_cache_read_tokens": 137.4, "avg_cache_write_tokens": 887.8, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 29.7, "median_query_seconds": 16.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 77.262, "latency": 11.318, "stderr": 1.738, "cost_per_test": 0.00113, "avg_input_tokens": 1052.6, "avg_output_tokens": 402.5, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 11.3, "median_query_seconds": 10.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 77.139, "latency": 8.366, "stderr": 1.468, "cost_per_test": 0.001022, "avg_input_tokens": 1062.0, "avg_output_tokens": 1067.0, "avg_reasoning_tokens": 785.7, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 8.4, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 77.139, "latency": 19.731, "stderr": 1.586, "cost_per_test": 0.008846, "avg_input_tokens": 1062.3, "avg_output_tokens": 1372.5, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 19.7, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 76.773, "latency": 77.672, "stderr": 1.492, "cost_per_test": 0.001618, "avg_input_tokens": 1100.2, "avg_output_tokens": 2171.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 77.7, "median_query_seconds": 47.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 76.65, "latency": 22.284, "stderr": 1.479, "cost_per_test": 0.001227, "avg_input_tokens": 1062.3, "avg_output_tokens": 300.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 22.3, "median_query_seconds": 20.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 76.528, "latency": 5.636, "stderr": 1.482, "cost_per_test": 0.005575, "avg_input_tokens": 1097.4, "avg_output_tokens": 283.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 5.6, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 76.406, "latency": 8.07, "stderr": 1.485, "cost_per_test": 0.001042, "avg_input_tokens": 1002.7, "avg_output_tokens": 1482.3, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 8.1, "median_query_seconds": 7.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 76.406, "latency": 8.799, "stderr": 1.485, "cost_per_test": 0.004377, "avg_input_tokens": 1096.4, "avg_output_tokens": 720.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 8.8, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 76.284, "latency": 5.289, "stderr": 1.487, "cost_per_test": 0.000695, "avg_input_tokens": 1074.4, "avg_output_tokens": 521.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 5.3, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 75.917, "latency": 2.117, "stderr": 1.495, "cost_per_test": 0.000209, "avg_input_tokens": 1062.0, "avg_output_tokens": 256.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 2.1, "median_query_seconds": 1.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 75.795, "latency": 40.606, "stderr": 1.498, "cost_per_test": 0.001217, "avg_input_tokens": 1164.4, "avg_output_tokens": 1737.1, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 40.6, "median_query_seconds": 33.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 75.672, "latency": 2.805, "stderr": 1.5, "cost_per_test": 0.000979, "avg_input_tokens": 1097.4, "avg_output_tokens": 337.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 2.8, "median_query_seconds": 2.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 75.306, "latency": 6.075, "stderr": 1.525, "cost_per_test": 0.005606, "avg_input_tokens": 1097.4, "avg_output_tokens": 286.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 6.1, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 75.306, "latency": 57.996, "stderr": 1.508, "cost_per_test": 0.023976, "avg_input_tokens": 1064.3, "avg_output_tokens": 2984.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 810, "avg_turns": 1.0, "avg_query_seconds": 58.0, "median_query_seconds": 54.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 75.061, "latency": 2.435, "stderr": 1.513, "cost_per_test": 0.000374, "avg_input_tokens": 1062.0, "avg_output_tokens": 669.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 2.4, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 75.061, "latency": 7.426, "stderr": 1.513, "cost_per_test": 0.000241, "avg_input_tokens": 1062.0, "avg_output_tokens": 1417.1, "avg_reasoning_tokens": 1079.9, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 7.4, "median_query_seconds": 7.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 75.061, "latency": 31.091, "stderr": 1.513, "cost_per_test": 0.001536, "avg_input_tokens": 1107.0, "avg_output_tokens": 1469.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 31.1, "median_query_seconds": 18.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 74.939, "latency": 4.505, "stderr": 1.57, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 74.817, "latency": 5.44, "stderr": 1.518, "cost_per_test": 0.000791, "avg_input_tokens": 1002.7, "avg_output_tokens": 980.9, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 5.4, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 74.817, "latency": 16.969, "stderr": 1.518, "cost_per_test": 0.008671, "avg_input_tokens": 1052.6, "avg_output_tokens": 1313.1, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 17.0, "median_query_seconds": 11.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 74.572, "latency": 5.016, "stderr": 1.523, "cost_per_test": 0.001232, "avg_input_tokens": 1112.7, "avg_output_tokens": 2018.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 5.0, "median_query_seconds": 3.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 74.572, "latency": 18.211, "stderr": 1.523, "cost_per_test": 0.001068, "avg_input_tokens": 1062.3, "avg_output_tokens": 124.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 18.2, "median_query_seconds": 12.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 74.572, "latency": 19.554, "stderr": 1.743, "cost_per_test": 0.011481, "avg_input_tokens": 1197.0, "avg_output_tokens": 2056.7, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 19.6, "median_query_seconds": 13.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-2-1212": { "accuracy": 73.961, "latency": 3.946, "stderr": 1.534, "cost_per_test": 0.003919, "avg_input_tokens": 1069.8, "avg_output_tokens": 177.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 3.9, "median_query_seconds": 3.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 73.839, "latency": 20.723, "stderr": 1.546, "cost_per_test": null, "avg_input_tokens": 1119.9, "avg_output_tokens": 999.7, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 20.7, "median_query_seconds": 16.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 73.594, "latency": 1.964, "stderr": 1.541, "cost_per_test": 0.000149, "avg_input_tokens": 1163.7, "avg_output_tokens": 123.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 970.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 2.0, "median_query_seconds": 1.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 73.594, "latency": 2.18, "stderr": 1.544, "cost_per_test": 0.001843, "avg_input_tokens": 1062.0, "avg_output_tokens": 103.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 2.2, "median_query_seconds": 1.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 73.227, "latency": 30.193, "stderr": 1.548, "cost_per_test": 0.001306, "avg_input_tokens": 1096.4, "avg_output_tokens": 3129.6, "avg_reasoning_tokens": 3027.2, "avg_cache_read_tokens": 13.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 30.2, "median_query_seconds": 25.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 73.227, "latency": 81.138, "stderr": 1.574, "cost_per_test": 0.000919, "avg_input_tokens": 1175.3, "avg_output_tokens": 1964.2, "avg_reasoning_tokens": 1728.1, "avg_cache_read_tokens": 1013.3, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 81.1, "median_query_seconds": 51.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 73.105, "latency": 4.147, "stderr": 1.55, "cost_per_test": 0.001362, "avg_input_tokens": 1126.9, "avg_output_tokens": 420.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 4.1, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 72.861, "latency": 4.163, "stderr": 1.555, "cost_per_test": 0.003581, "avg_input_tokens": 1228.7, "avg_output_tokens": 187.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 4.2, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 72.127, "latency": 5.585, "stderr": 1.674, "cost_per_test": 0.000201, "avg_input_tokens": 1097.3, "avg_output_tokens": 365.3, "avg_reasoning_tokens": 272.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 5.6, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 71.883, "latency": 4.621, "stderr": 1.572, "cost_per_test": 0.000437, "avg_input_tokens": 1074.4, "avg_output_tokens": 412.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 4.6, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 70.782, "latency": 2.018, "stderr": 1.59, "cost_per_test": 0.000134, "avg_input_tokens": 1052.6, "avg_output_tokens": 181.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 2.0, "median_query_seconds": 1.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 70.538, "latency": 6.736, "stderr": 1.683, "cost_per_test": 0.006182, "avg_input_tokens": 1088.9, "avg_output_tokens": 346.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 6.7, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 69.682, "latency": 1.938, "stderr": 1.626, "cost_per_test": 0.000255, "avg_input_tokens": 1097.4, "avg_output_tokens": 363.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 1.9, "median_query_seconds": 1.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 69.56, "latency": 2.709, "stderr": 1.609, "cost_per_test": 0.000342, "avg_input_tokens": 1052.6, "avg_output_tokens": 218.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 2.7, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 68.826, "latency": 32.333, "stderr": 1.745, "cost_per_test": 0.000352, "avg_input_tokens": 1212.3, "avg_output_tokens": 1415.6, "avg_reasoning_tokens": 1172.6, "avg_cache_read_tokens": 1040.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 32.3, "median_query_seconds": 24.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 68.46, "latency": 45.25, "stderr": 1.673, "cost_per_test": 0.000695, "avg_input_tokens": 1164.4, "avg_output_tokens": 2043.4, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 45.3, "median_query_seconds": 37.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 67.848, "latency": 11.844, "stderr": 1.709, "cost_per_test": null, "avg_input_tokens": 1119.9, "avg_output_tokens": 646.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 11.8, "median_query_seconds": 11.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 67.482, "latency": 3.66, "stderr": 1.638, "cost_per_test": 0.000298, "avg_input_tokens": 1097.4, "avg_output_tokens": 222.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 3.7, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 67.237, "latency": 1.761, "stderr": 1.641, "cost_per_test": 0.00089, "avg_input_tokens": 1122.7, "avg_output_tokens": 1331.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 1.8, "median_query_seconds": 1.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 66.748, "latency": 5.851, "stderr": 1.647, "cost_per_test": 0.001951, "avg_input_tokens": 1169.0, "avg_output_tokens": 253.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 5.9, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 62.714, "latency": 1.396, "stderr": 1.691, "cost_per_test": 0.000135, "avg_input_tokens": 1062.0, "avg_output_tokens": 183.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 1.4, "median_query_seconds": 1.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 56.846, "latency": 7.652, "stderr": 1.732, "cost_per_test": 0.003323, "avg_input_tokens": 1151.1, "avg_output_tokens": 127.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 7.7, "median_query_seconds": 5.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 54.279, "latency": 8.939, "stderr": 1.742, "cost_per_test": 0.002333, "avg_input_tokens": 1052.6, "avg_output_tokens": 1204.2, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 8.9, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-r-plus": { "accuracy": 46.088, "latency": 5.687, "stderr": 1.743, "cost_per_test": 0.005504, "avg_input_tokens": 1088.9, "avg_output_tokens": 278.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 4.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 43.399, "latency": 1.08, "stderr": 1.733, "cost_per_test": 0.000264, "avg_input_tokens": 1151.1, "avg_output_tokens": 83.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 818, "avg_turns": 1.0, "avg_query_seconds": 1.1, "median_query_seconds": 0.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "history": { "anthropic/claude-opus-5": { "accuracy": 86.614, "latency": 8.395, "stderr": 1.744, "cost_per_test": 0.03216, "avg_input_tokens": 3173.0, "avg_output_tokens": 493.3, "avg_reasoning_tokens": 271.5, "avg_cache_write_tokens": 3171.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 8.4, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 85.564, "latency": 15.026, "stderr": 1.801, "cost_per_test": 0.018593, "avg_input_tokens": 2189.9, "avg_output_tokens": 1184.4, "avg_reasoning_tokens": 1010.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 15.0, "median_query_seconds": 9.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 85.302, "latency": 17.932, "stderr": 1.814, "cost_per_test": 0.099496, "avg_input_tokens": 3174.3, "avg_output_tokens": 1196.5, "avg_reasoning_tokens": 996.4, "avg_cache_write_tokens": 3170.3, "turn_metrics": { "tasks_measured": 380, "avg_turns": 1.0, "avg_query_seconds": 17.9, "median_query_seconds": 9.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 85.302, "latency": 32.513, "stderr": 1.814, "cost_per_test": 0.079154, "avg_input_tokens": 3173.0, "avg_output_tokens": 789.9, "avg_cache_write_tokens": 3171.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 32.5, "median_query_seconds": 10.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 84.777, "latency": 19.767, "stderr": 1.84, "cost_per_test": 0.029725, "avg_input_tokens": 2189.9, "avg_output_tokens": 2151.2, "avg_reasoning_tokens": 1879.3, "avg_cache_read_tokens": 260.9, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 19.8, "median_query_seconds": 13.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 83.99, "latency": 3.589, "stderr": 1.879, "cost_per_test": 0.00988, "avg_input_tokens": 2189.9, "avg_output_tokens": 879.4, "avg_reasoning_tokens": 751.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 3.6, "median_query_seconds": 2.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 83.727, "latency": 7.452, "stderr": 1.891, "cost_per_test": 0.020687, "avg_input_tokens": 2189.9, "avg_output_tokens": 2320.3, "avg_reasoning_tokens": 2157.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 7.5, "median_query_seconds": 3.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 83.465, "latency": 5.946, "stderr": 1.903, "cost_per_test": 0.011323, "avg_input_tokens": 2189.9, "avg_output_tokens": 1071.7, "avg_reasoning_tokens": 888.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 5.9, "median_query_seconds": 5.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 83.202, "latency": 134.626, "stderr": 1.962, "cost_per_test": 0.04407, "avg_input_tokens": 3178.0, "avg_output_tokens": 987.1, "avg_cache_read_tokens": 80.6, "avg_cache_write_tokens": 3091.5, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 134.6, "median_query_seconds": 89.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 82.677, "latency": 12.47, "stderr": 1.939, "cost_per_test": 0.04551, "avg_input_tokens": 3173.0, "avg_output_tokens": 1027.2, "avg_cache_write_tokens": 3171.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 12.5, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 82.677, "latency": 20.649, "stderr": 2.553, "cost_per_test": 0.022339, "avg_input_tokens": 2333.4, "avg_output_tokens": 1022.6, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 20.6, "median_query_seconds": 16.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 82.415, "latency": 20.301, "stderr": 1.95, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 82.152, "latency": 24.783, "stderr": 1.962, "cost_per_test": 0.015035, "avg_input_tokens": 2216.0, "avg_output_tokens": 1265.9, "avg_reasoning_tokens": 1026.5, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 24.8, "median_query_seconds": 23.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 81.89, "latency": 9.19, "stderr": 1.973, "cost_per_test": 0.021669, "avg_input_tokens": 2142.8, "avg_output_tokens": 441.8, "avg_reasoning_tokens": 395.5, "avg_cache_read_tokens": 510.7, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 9.2, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 81.89, "latency": 21.033, "stderr": 1.973, "cost_per_test": 0.102264, "avg_input_tokens": 2344.4, "avg_output_tokens": 894.6, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 21.0, "median_query_seconds": 20.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 81.89, "latency": 29.478, "stderr": 2.159, "cost_per_test": 0.029215, "avg_input_tokens": 2344.4, "avg_output_tokens": 1478.8, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 29.5, "median_query_seconds": 21.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 81.89, "latency": 60.188, "stderr": 1.973, "cost_per_test": 0.008557, "avg_input_tokens": 2145.6, "avg_output_tokens": 1798.9, "avg_reasoning_tokens": 1590.0, "avg_cache_read_tokens": 1609.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 60.2, "median_query_seconds": 46.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 81.627, "latency": 11.228, "stderr": 1.984, "cost_per_test": 0.021196, "avg_input_tokens": 2189.9, "avg_output_tokens": 1401.4, "avg_reasoning_tokens": 1204.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 11.2, "median_query_seconds": 9.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 81.627, "latency": 58.094, "stderr": 1.984, "cost_per_test": 0.025591, "avg_input_tokens": 2141.0, "avg_output_tokens": 1088.0, "avg_reasoning_tokens": 1021.0, "avg_cache_read_tokens": 153.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 58.1, "median_query_seconds": 36.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 81.365, "latency": 427.645, "stderr": 1.995, "cost_per_test": 0.011201, "avg_input_tokens": 2154.5, "avg_output_tokens": 1588.1, "avg_reasoning_tokens": 1391.0, "avg_cache_read_tokens": 1757.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 427.6, "median_query_seconds": 411.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 81.102, "latency": 24.579, "stderr": 2.282, "cost_per_test": 0.030658, "avg_input_tokens": 2333.4, "avg_output_tokens": 1460.7, "avg_cache_write_tokens": 2330.4, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 24.6, "median_query_seconds": 15.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 81.102, "latency": 38.219, "stderr": 2.006, "cost_per_test": 0.007206, "avg_input_tokens": 2215.7, "avg_output_tokens": 2032.8, "avg_reasoning_tokens": 1860.7, "turn_metrics": { "tasks_measured": 380, "avg_turns": 1.0, "avg_query_seconds": 38.2, "median_query_seconds": 36.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 81.102, "latency": 40.614, "stderr": 2.006, "cost_per_test": 0.013065, "avg_input_tokens": 2254.0, "avg_output_tokens": 1720.2, "avg_reasoning_tokens": 1620.9, "avg_cache_read_tokens": 1007.9, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 40.6, "median_query_seconds": 13.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 81.102, "latency": 47.917, "stderr": 2.006, "cost_per_test": 0.01487, "avg_input_tokens": 2154.5, "avg_output_tokens": 2145.7, "avg_reasoning_tokens": 1994.2, "avg_cache_read_tokens": 1541.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 47.9, "median_query_seconds": 23.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 80.84, "latency": 27.069, "stderr": 2.016, "cost_per_test": 0.013377, "avg_input_tokens": 2189.9, "avg_output_tokens": 4147.6, "avg_reasoning_tokens": 3876.0, "avg_cache_read_tokens": 356.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 27.1, "median_query_seconds": 17.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark": { "accuracy": 80.84, "latency": 47.223, "stderr": 2.016, "cost_per_test": 0.001956, "avg_input_tokens": 2200.6, "avg_output_tokens": 2709.9, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 47.2, "median_query_seconds": 48.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 80.315, "latency": 11.317, "stderr": 2.047, "cost_per_test": 0.059706, "avg_input_tokens": 2316.4, "avg_output_tokens": 332.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 11.3, "median_query_seconds": 11.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 80.315, "latency": 23.848, "stderr": 2.115, "cost_per_test": 0.132189, "avg_input_tokens": 2345.4, "avg_output_tokens": 1293.4, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 23.8, "median_query_seconds": 19.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 80.315, "latency": 24.241, "stderr": 2.077, "cost_per_test": 0.046377, "avg_input_tokens": 3173.0, "avg_output_tokens": 2298.6, "avg_cache_write_tokens": 3171.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 24.2, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 79.79, "latency": 16.33, "stderr": 2.057, "cost_per_test": 0.00874, "avg_input_tokens": 2302.6, "avg_output_tokens": 1814.3, "avg_reasoning_tokens": 1646.7, "avg_cache_read_tokens": 1681.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 16.3, "median_query_seconds": 13.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 79.528, "latency": 10.208, "stderr": 2.067, "cost_per_test": 0.006033, "avg_input_tokens": 2141.0, "avg_output_tokens": 232.2, "avg_reasoning_tokens": 186.5, "avg_cache_read_tokens": 575.2, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 10.2, "median_query_seconds": 5.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 79.528, "latency": 38.904, "stderr": 2.067, "cost_per_test": 0.003043, "avg_input_tokens": 2129.0, "avg_output_tokens": 1879.9, "avg_cache_read_tokens": 31.0, "avg_cache_write_tokens": 2096.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 38.9, "median_query_seconds": 20.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 79.265, "latency": 10.566, "stderr": 2.077, "cost_per_test": 0.012016, "avg_input_tokens": 2316.4, "avg_output_tokens": 337.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 10.6, "median_query_seconds": 8.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 79.265, "latency": 14.649, "stderr": 2.077, "cost_per_test": 0.01165, "avg_input_tokens": 2239.8, "avg_output_tokens": 691.5, "avg_reasoning_tokens": 529.9, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 14.6, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 79.265, "latency": 20.393, "stderr": 2.4, "cost_per_test": 0.040285, "avg_input_tokens": 2141.0, "avg_output_tokens": 1197.5, "avg_reasoning_tokens": 1122.9, "avg_cache_read_tokens": 505.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 20.4, "median_query_seconds": 11.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 79.265, "latency": 23.049, "stderr": 2.077, "cost_per_test": 0.01705, "avg_input_tokens": 2141.0, "avg_output_tokens": 1437.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 23.0, "median_query_seconds": 18.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 78.74, "latency": 75.191, "stderr": 2.096, "cost_per_test": 0.008337, "avg_input_tokens": 2215.8, "avg_output_tokens": 3104.4, "avg_reasoning_tokens": 2776.7, "turn_metrics": { "tasks_measured": 380, "avg_turns": 1.0, "avg_query_seconds": 75.2, "median_query_seconds": 61.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 78.478, "latency": 3.845, "stderr": 2.105, "cost_per_test": 0.003345, "avg_input_tokens": 2189.9, "avg_output_tokens": 1075.1, "avg_reasoning_tokens": 911.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 3.8, "median_query_seconds": 3.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 78.478, "latency": 8.614, "stderr": 2.124, "cost_per_test": 0.061382, "avg_input_tokens": 2316.4, "avg_output_tokens": 355.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 8.6, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 78.478, "latency": 36.299, "stderr": 2.115, "cost_per_test": 0.006593, "avg_input_tokens": 2220.0, "avg_output_tokens": 1547.0, "avg_reasoning_tokens": 1479.6, "avg_cache_read_tokens": 1930.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 36.3, "median_query_seconds": 12.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 78.478, "latency": 38.643, "stderr": 2.105, "cost_per_test": 0.010747, "avg_input_tokens": 2151.4, "avg_output_tokens": 1965.4, "avg_reasoning_tokens": 1787.0, "avg_cache_read_tokens": 800.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 38.6, "median_query_seconds": 12.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 78.478, "latency": 96.434, "stderr": 2.105, "cost_per_test": 0.011629, "avg_input_tokens": 2161.2, "avg_output_tokens": 2738.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1746.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 380, "avg_turns": 1.0, "avg_query_seconds": 96.4, "median_query_seconds": 49.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.1": { "accuracy": 78.421, "latency": 32.724, "stderr": 2.11, "cost_per_test": 0.006012, "avg_input_tokens": 2144.1, "avg_output_tokens": 1664.0, "avg_reasoning_tokens": 1408.7, "avg_cache_read_tokens": 1821.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 380, "avg_turns": 1.0, "avg_query_seconds": 32.7, "median_query_seconds": 19.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 78.215, "latency": 36.902, "stderr": 2.115, "cost_per_test": 0.000893, "avg_input_tokens": 2189.9, "avg_output_tokens": 1685.7, "avg_reasoning_tokens": 1462.4, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 36.9, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 77.953, "latency": 9.482, "stderr": 2.124, "cost_per_test": 0.009669, "avg_input_tokens": 2141.0, "avg_output_tokens": 673.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 9.5, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 77.953, "latency": 3277.635, "stderr": 2.124, "cost_per_test": 0.007354, "avg_input_tokens": 2097.1, "avg_output_tokens": 2606.0, "avg_reasoning_tokens": 2330.5, "avg_cache_read_tokens": 1697.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 3277.6, "median_query_seconds": 3205.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 77.69, "latency": 22.006, "stderr": 2.133, "cost_per_test": 0.009824, "avg_input_tokens": 2141.0, "avg_output_tokens": 897.3, "avg_reasoning_tokens": 804.6, "avg_cache_read_tokens": 1622.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 22.0, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 77.69, "latency": 635.878, "stderr": 2.193, "cost_per_test": 0.010836, "avg_input_tokens": 2207.0, "avg_output_tokens": 2579.6, "avg_reasoning_tokens": 2464.1, "avg_cache_read_tokens": 1797.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 635.9, "median_query_seconds": 616.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 77.428, "latency": 7.596, "stderr": 2.142, "cost_per_test": 0.010326, "avg_input_tokens": 2208.1, "avg_output_tokens": 1529.0, "avg_reasoning_tokens": 1339.1, "avg_cache_read_tokens": 1813.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 7.6, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 77.428, "latency": 57.957, "stderr": 2.142, "cost_per_test": 0.007885, "avg_input_tokens": 2144.9, "avg_output_tokens": 2219.8, "avg_cache_read_tokens": 1651.7, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 58.0, "median_query_seconds": 42.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 77.165, "latency": 6.155, "stderr": 2.151, "cost_per_test": 0.011353, "avg_input_tokens": 2316.4, "avg_output_tokens": 293.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 6.2, "median_query_seconds": 6.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 77.165, "latency": 10.586, "stderr": 2.151, "cost_per_test": 0.001394, "avg_input_tokens": 2207.0, "avg_output_tokens": 942.1, "avg_reasoning_tokens": 891.8, "avg_cache_read_tokens": 1927.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 10.6, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 76.903, "latency": 38.522, "stderr": 2.159, "cost_per_test": 0.010321, "avg_input_tokens": 2151.4, "avg_output_tokens": 1885.8, "avg_reasoning_tokens": 1717.1, "avg_cache_read_tokens": 867.4, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 38.5, "median_query_seconds": 20.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 76.903, "latency": 47.346, "stderr": 2.159, "cost_per_test": 0.003762, "avg_input_tokens": 2149.0, "avg_output_tokens": 2975.2, "avg_cache_read_tokens": 1886.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 47.3, "median_query_seconds": 13.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 76.64, "latency": 39.88, "stderr": 2.526, "cost_per_test": 0.001114, "avg_input_tokens": 2428.3, "avg_output_tokens": 1101.4, "avg_reasoning_tokens": 931.7, "avg_cache_read_tokens": 2088.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 39.9, "median_query_seconds": 27.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 76.378, "latency": 9.309, "stderr": 2.176, "cost_per_test": 0.001366, "avg_input_tokens": 2171.9, "avg_output_tokens": 1174.0, "avg_reasoning_tokens": 888.3, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 9.3, "median_query_seconds": 9.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 76.378, "latency": 15.675, "stderr": 2.176, "cost_per_test": 0.009111, "avg_input_tokens": 2171.9, "avg_output_tokens": 639.6, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 15.7, "median_query_seconds": 14.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 76.378, "latency": 21.72, "stderr": 2.201, "cost_per_test": 0.020553, "avg_input_tokens": 2123.7, "avg_output_tokens": 2599.4, "avg_reasoning_tokens": 1259.2, "avg_cache_read_tokens": 1222.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 21.7, "median_query_seconds": 15.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 76.378, "latency": 27.199, "stderr": 2.176, "cost_per_test": 0.005502, "avg_input_tokens": 2161.2, "avg_output_tokens": 1692.7, "avg_cache_read_tokens": 1746.2, "turn_metrics": { "tasks_measured": 380, "avg_turns": 1.0, "avg_query_seconds": 27.2, "median_query_seconds": 21.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 76.115, "latency": 13.577, "stderr": 2.184, "cost_per_test": null, "avg_input_tokens": 2205.7, "avg_output_tokens": 1289.7, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 13.6, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 76.115, "latency": 21.861, "stderr": 2.216, "cost_per_test": null, "avg_input_tokens": 2167.2, "avg_output_tokens": 1075.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 21.9, "median_query_seconds": 21.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 76.115, "latency": 43.959, "stderr": 2.184, "cost_per_test": 0.023192, "avg_input_tokens": 2149.0, "avg_output_tokens": 2212.6, "avg_cache_read_tokens": 1856.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 44.0, "median_query_seconds": 12.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 76.115, "latency": 61.056, "stderr": 2.184, "cost_per_test": 0.000701, "avg_input_tokens": 2128.3, "avg_output_tokens": 1314.1, "avg_reasoning_tokens": 1243.0, "avg_cache_read_tokens": 1760.4, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 61.1, "median_query_seconds": 50.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 75.853, "latency": 10.163, "stderr": 2.209, "cost_per_test": 0.001759, "avg_input_tokens": 2141.0, "avg_output_tokens": 1109.1, "avg_reasoning_tokens": 1056.1, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 10.2, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 75.853, "latency": 15.211, "stderr": 2.193, "cost_per_test": 0.000752, "avg_input_tokens": 2240.1, "avg_output_tokens": 1166.0, "avg_reasoning_tokens": 1064.3, "avg_cache_read_tokens": 1859.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 15.2, "median_query_seconds": 9.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 75.591, "latency": 18.797, "stderr": 2.201, "cost_per_test": 0.006689, "avg_input_tokens": 2258.0, "avg_output_tokens": 1853.3, "avg_reasoning_tokens": 1752.2, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 18.8, "median_query_seconds": 7.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 75.591, "latency": 31.974, "stderr": 2.201, "cost_per_test": 0.029222, "avg_input_tokens": 2344.4, "avg_output_tokens": 1479.2, "turn_metrics": { "tasks_measured": 379, "avg_turns": 1.0, "avg_query_seconds": 32.0, "median_query_seconds": 25.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-0709": { "accuracy": 75.591, "latency": 50.819, "stderr": 2.201, "cost_per_test": 0.032609, "avg_input_tokens": 2091.1, "avg_output_tokens": 1755.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 50.8, "median_query_seconds": 24.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 75.328, "latency": 6.513, "stderr": 2.209, "cost_per_test": 0.010784, "avg_input_tokens": 2316.4, "avg_output_tokens": 255.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 6.5, "median_query_seconds": 6.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 75.328, "latency": 15.341, "stderr": 2.209, "cost_per_test": 0.002869, "avg_input_tokens": 2141.0, "avg_output_tokens": 1275.3, "avg_reasoning_tokens": 1208.5, "avg_cache_read_tokens": 962.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 15.3, "median_query_seconds": 9.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 75.328, "latency": 63.49, "stderr": 2.216, "cost_per_test": 0.000199, "avg_input_tokens": 2128.0, "avg_output_tokens": 312.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 2094.4, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 63.5, "median_query_seconds": 45.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 75.066, "latency": 23.108, "stderr": 2.216, "cost_per_test": 0.103124, "avg_input_tokens": 2141.0, "avg_output_tokens": 1183.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 23.1, "median_query_seconds": 17.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 74.803, "latency": 5.53, "stderr": 2.224, "cost_per_test": 0.001343, "avg_input_tokens": 2171.9, "avg_output_tokens": 1153.8, "avg_reasoning_tokens": 877.3, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 5.5, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 74.803, "latency": 21.322, "stderr": 2.224, "cost_per_test": 0.000572, "avg_input_tokens": 2151.4, "avg_output_tokens": 1905.5, "avg_cache_read_tokens": 1095.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 21.3, "median_query_seconds": 7.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 74.803, "latency": 89.177, "stderr": 2.224, "cost_per_test": 0.02017, "avg_input_tokens": 2186.6, "avg_output_tokens": 2924.3, "avg_reasoning_tokens": 2426.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 380, "avg_turns": 1.0, "avg_query_seconds": 89.2, "median_query_seconds": 67.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max": { "accuracy": 74.278, "latency": 0.0, "stderr": 2.239, "cost_per_test": 0.005804, "avg_input_tokens": 2185.3, "avg_output_tokens": 530.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 74.278, "latency": 19.362, "stderr": 2.239, "cost_per_test": 0.024612, "avg_input_tokens": 2344.4, "avg_output_tokens": 1171.9, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 19.4, "median_query_seconds": 16.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 74.016, "latency": 4.846, "stderr": 2.247, "cost_per_test": 0.001969, "avg_input_tokens": 2201.1, "avg_output_tokens": 3057.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 4.8, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 74.016, "latency": 16.728, "stderr": 2.385, "cost_per_test": 0.009046, "avg_input_tokens": 2127.0, "avg_output_tokens": 1143.5, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 16.7, "median_query_seconds": 12.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 73.491, "latency": 30.172, "stderr": 2.261, "cost_per_test": 0.005582, "avg_input_tokens": 2185.3, "avg_output_tokens": 493.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 30.2, "median_query_seconds": 27.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 73.228, "latency": 25.596, "stderr": 2.268, "cost_per_test": 0.00265, "avg_input_tokens": 2267.0, "avg_output_tokens": 921.9, "avg_cache_read_tokens": 1922.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 25.6, "median_query_seconds": 18.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-4.7": { "accuracy": 72.703, "latency": 61.031, "stderr": 2.282, "cost_per_test": 0.005314, "avg_input_tokens": 2144.4, "avg_output_tokens": 2180.1, "avg_cache_read_tokens": 1568.5, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 61.0, "median_query_seconds": 39.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 72.178, "latency": 3.257, "stderr": 2.296, "cost_per_test": 0.006572, "avg_input_tokens": 2142.0, "avg_output_tokens": 285.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 3.3, "median_query_seconds": 3.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 72.178, "latency": 87.832, "stderr": 2.302, "cost_per_test": 0.002134, "avg_input_tokens": 2161.4, "avg_output_tokens": 2118.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 87.8, "median_query_seconds": 53.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 71.916, "latency": 6.494, "stderr": 2.302, "cost_per_test": 0.007865, "avg_input_tokens": 2142.0, "avg_output_tokens": 251.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 6.5, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 71.654, "latency": 14.023, "stderr": 2.309, "cost_per_test": 0.003525, "avg_input_tokens": 2141.0, "avg_output_tokens": 1494.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 14.0, "median_query_seconds": 11.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 71.654, "latency": 37.78, "stderr": 2.322, "cost_per_test": 0.001624, "avg_input_tokens": 2185.3, "avg_output_tokens": 1299.6, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 37.8, "median_query_seconds": 21.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 71.391, "latency": 36.51, "stderr": 2.315, "cost_per_test": 0.001704, "avg_input_tokens": 2215.7, "avg_output_tokens": 3705.0, "avg_reasoning_tokens": 3338.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 379, "avg_turns": 1.0, "avg_query_seconds": 36.5, "median_query_seconds": 30.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-4.5": { "accuracy": 71.129, "latency": 64.589, "stderr": 2.322, "cost_per_test": 0.00669, "avg_input_tokens": 2144.4, "avg_output_tokens": 2456.3, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 64.6, "median_query_seconds": 57.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-2-1212": { "accuracy": 70.866, "latency": 3.751, "stderr": 2.328, "cost_per_test": 0.006042, "avg_input_tokens": 2148.5, "avg_output_tokens": 174.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 3.8, "median_query_seconds": 3.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 70.866, "latency": 7.315, "stderr": 2.328, "cost_per_test": 0.005467, "avg_input_tokens": 2141.0, "avg_output_tokens": 707.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 7.3, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 70.866, "latency": 11.778, "stderr": 2.555, "cost_per_test": 0.001663, "avg_input_tokens": 2192.6, "avg_output_tokens": 381.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 11.8, "median_query_seconds": 10.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-3": { "accuracy": 70.604, "latency": 11.052, "stderr": 2.334, "cost_per_test": 0.017408, "avg_input_tokens": 2090.1, "avg_output_tokens": 742.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 11.1, "median_query_seconds": 10.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 70.341, "latency": 18.203, "stderr": 2.561, "cost_per_test": 0.012087, "avg_input_tokens": 2344.4, "avg_output_tokens": 1948.5, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 18.2, "median_query_seconds": 12.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 70.079, "latency": 7.615, "stderr": 2.346, "cost_per_test": 0.002016, "avg_input_tokens": 2127.0, "avg_output_tokens": 112.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 7.6, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 69.816, "latency": 7.998, "stderr": 2.352, "cost_per_test": 0.001357, "avg_input_tokens": 2091.1, "avg_output_tokens": 1458.3, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 8.0, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 69.816, "latency": 28.434, "stderr": 2.352, "cost_per_test": 0.000734, "avg_input_tokens": 2148.6, "avg_output_tokens": 297.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 28.4, "median_query_seconds": 2.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 69.554, "latency": 4.636, "stderr": 2.358, "cost_per_test": 0.000237, "avg_input_tokens": 2241.7, "avg_output_tokens": 792.7, "avg_cache_read_tokens": 1763.8, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 4.6, "median_query_seconds": 3.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 69.554, "latency": 52.52, "stderr": 2.358, "cost_per_test": 0.030174, "avg_input_tokens": 2204.7, "avg_output_tokens": 3582.2, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 52.5, "median_query_seconds": 50.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 69.291, "latency": 14.335, "stderr": 2.363, "cost_per_test": 0.003214, "avg_input_tokens": 2170.8, "avg_output_tokens": 347.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 14.3, "median_query_seconds": 13.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 69.029, "latency": 1.28, "stderr": 2.369, "cost_per_test": 0.000268, "avg_input_tokens": 2171.9, "avg_output_tokens": 128.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 1.3, "median_query_seconds": 0.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 69.029, "latency": 5.387, "stderr": 2.369, "cost_per_test": 0.001109, "avg_input_tokens": 2091.1, "avg_output_tokens": 963.4, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 5.4, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 68.241, "latency": 16.687, "stderr": 2.39, "cost_per_test": 0.000427, "avg_input_tokens": 2424.3, "avg_output_tokens": 1289.2, "avg_reasoning_tokens": 1171.5, "avg_cache_read_tokens": 1990.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 16.7, "median_query_seconds": 12.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 68.241, "latency": 29.697, "stderr": 2.385, "cost_per_test": 0.001333, "avg_input_tokens": 2209.0, "avg_output_tokens": 1669.2, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 29.7, "median_query_seconds": 26.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 67.979, "latency": 2.571, "stderr": 2.39, "cost_per_test": 0.001981, "avg_input_tokens": 2127.0, "avg_output_tokens": 73.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 2.6, "median_query_seconds": 2.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 67.979, "latency": 2.792, "stderr": 2.39, "cost_per_test": 0.000524, "avg_input_tokens": 2171.9, "avg_output_tokens": 767.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 2.8, "median_query_seconds": 2.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 67.979, "latency": 3.949, "stderr": 2.405, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-4.6": { "accuracy": 67.979, "latency": 36.461, "stderr": 2.39, "cost_per_test": 0.005921, "avg_input_tokens": 2146.1, "avg_output_tokens": 2106.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 36.5, "median_query_seconds": 28.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 67.454, "latency": 34.015, "stderr": 2.4, "cost_per_test": 0.004019, "avg_input_tokens": 2132.0, "avg_output_tokens": 1346.6, "avg_cache_read_tokens": 36.0, "avg_cache_write_tokens": 2096.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 34.0, "median_query_seconds": 26.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 67.192, "latency": 1.967, "stderr": 2.405, "cost_per_test": 0.003125, "avg_input_tokens": 2171.9, "avg_output_tokens": 82.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 2.0, "median_query_seconds": 1.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 67.192, "latency": 3.02, "stderr": 2.405, "cost_per_test": 0.001395, "avg_input_tokens": 2142.0, "avg_output_tokens": 336.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 3.0, "median_query_seconds": 2.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 66.404, "latency": 8.115, "stderr": 2.42, "cost_per_test": 0.000345, "avg_input_tokens": 2171.9, "avg_output_tokens": 1396.2, "avg_reasoning_tokens": 1076.6, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 8.1, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 66.142, "latency": 2.078, "stderr": 2.424, "cost_per_test": 0.000239, "avg_input_tokens": 2252.1, "avg_output_tokens": 114.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1790.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 2.1, "median_query_seconds": 1.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 65.879, "latency": 20.203, "stderr": 2.429, "cost_per_test": 0.012645, "avg_input_tokens": 2141.0, "avg_output_tokens": 2338.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 20.2, "median_query_seconds": 15.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 65.617, "latency": 5.575, "stderr": 2.54, "cost_per_test": 0.008195, "avg_input_tokens": 2142.0, "avg_output_tokens": 283.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 5.6, "median_query_seconds": 4.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 65.617, "latency": 29.914, "stderr": 2.438, "cost_per_test": 0.002667, "avg_input_tokens": 2112.0, "avg_output_tokens": 1566.9, "avg_cache_read_tokens": 16.0, "avg_cache_write_tokens": 2096.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 29.9, "median_query_seconds": 17.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 65.092, "latency": 1.88, "stderr": 2.442, "cost_per_test": 0.00206, "avg_input_tokens": 2174.2, "avg_output_tokens": 166.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 1.9, "median_query_seconds": 1.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 65.092, "latency": 18.999, "stderr": 2.442, "cost_per_test": 0.011342, "avg_input_tokens": 2192.7, "avg_output_tokens": 1391.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 19.0, "median_query_seconds": 13.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 64.829, "latency": 19.588, "stderr": 2.446, "cost_per_test": null, "avg_input_tokens": 2167.2, "avg_output_tokens": 882.5, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 19.6, "median_query_seconds": 17.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 64.042, "latency": 3.039, "stderr": 2.458, "cost_per_test": 0.005579, "avg_input_tokens": 2388.5, "avg_output_tokens": 133.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 3.0, "median_query_seconds": 2.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 64.042, "latency": 5.903, "stderr": 2.474, "cost_per_test": 0.009169, "avg_input_tokens": 2173.3, "avg_output_tokens": 373.5, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 5.9, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 63.255, "latency": 5.903, "stderr": 2.47, "cost_per_test": 0.00281, "avg_input_tokens": 2316.4, "avg_output_tokens": 239.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 5.9, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 63.089, "latency": 2.238, "stderr": 2.469, "cost_per_test": 0.001623, "avg_input_tokens": 2210.7, "avg_output_tokens": 2361.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 2.2, "median_query_seconds": 1.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 61.942, "latency": 38.253, "stderr": 2.487, "cost_per_test": 0.001643, "avg_input_tokens": 2141.0, "avg_output_tokens": 4002.2, "avg_reasoning_tokens": 3875.9, "avg_cache_read_tokens": 1436.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 38.3, "median_query_seconds": 31.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 61.68, "latency": 18.353, "stderr": 2.5, "cost_per_test": 0.000833, "avg_input_tokens": 2209.0, "avg_output_tokens": 2262.1, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 18.4, "median_query_seconds": 16.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 61.68, "latency": 27.185, "stderr": 2.524, "cost_per_test": 0.000802, "avg_input_tokens": 2270.8, "avg_output_tokens": 1371.1, "avg_reasoning_tokens": 1171.2, "avg_cache_read_tokens": 2002.7, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 27.2, "median_query_seconds": 20.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 60.367, "latency": 30.011, "stderr": 2.561, "cost_per_test": 0.000367, "avg_input_tokens": 2307.8, "avg_output_tokens": 1176.7, "avg_reasoning_tokens": 956.1, "avg_cache_read_tokens": 1974.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 30.0, "median_query_seconds": 25.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 59.58, "latency": 1.878, "stderr": 2.514, "cost_per_test": 0.00021, "avg_input_tokens": 2192.7, "avg_output_tokens": 151.5, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 1.9, "median_query_seconds": 1.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 59.318, "latency": 4.698, "stderr": 2.54, "cost_per_test": 0.000241, "avg_input_tokens": 2141.0, "avg_output_tokens": 352.9, "avg_reasoning_tokens": 261.4, "avg_cache_read_tokens": 169.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 3.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 58.53, "latency": 1.045, "stderr": 2.524, "cost_per_test": 0.000199, "avg_input_tokens": 2171.9, "avg_output_tokens": 121.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 1.0, "median_query_seconds": 0.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 58.53, "latency": 5.104, "stderr": 2.524, "cost_per_test": 0.000675, "avg_input_tokens": 2148.6, "avg_output_tokens": 489.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 5.1, "median_query_seconds": 5.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 58.005, "latency": 8.725, "stderr": 2.529, "cost_per_test": 0.002683, "avg_input_tokens": 2192.7, "avg_output_tokens": 1057.7, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 8.7, "median_query_seconds": 6.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 55.381, "latency": 1.88, "stderr": 2.547, "cost_per_test": 0.000367, "avg_input_tokens": 2142.0, "avg_output_tokens": 380.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 1.9, "median_query_seconds": 1.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 54.593, "latency": 2.482, "stderr": 2.551, "cost_per_test": 0.00042, "avg_input_tokens": 2142.0, "avg_output_tokens": 164.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 2.5, "median_query_seconds": 1.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 54.593, "latency": 3.661, "stderr": 2.551, "cost_per_test": 0.000569, "avg_input_tokens": 2192.7, "avg_output_tokens": 217.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 3.7, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-r-plus": { "accuracy": 49.344, "latency": 1.934, "stderr": 2.561, "cost_per_test": 0.00626, "avg_input_tokens": 2173.3, "avg_output_tokens": 82.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 1.9, "median_query_seconds": 1.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 34.121, "latency": 1.15, "stderr": 2.429, "cost_per_test": 0.000471, "avg_input_tokens": 2266.7, "avg_output_tokens": 45.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 1.1, "median_query_seconds": 1.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 28.871, "latency": 3.054, "stderr": 2.322, "cost_per_test": 0.004932, "avg_input_tokens": 2266.7, "avg_output_tokens": 49.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 381, "avg_turns": 1.0, "avg_query_seconds": 3.1, "median_query_seconds": 1.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "law": { "anthropic/claude-fable-5-1": { "accuracy": 87.466, "latency": 22.156, "stderr": 0.998, "cost_per_test": 0.105418, "avg_input_tokens": 2246.2, "avg_output_tokens": 1554.3, "avg_reasoning_tokens": 1262.7, "avg_cache_read_tokens": 29.8, "avg_cache_write_tokens": 2212.4, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 22.2, "median_query_seconds": 12.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 86.558, "latency": 11.668, "stderr": 1.028, "cost_per_test": 0.032255, "avg_input_tokens": 2244.2, "avg_output_tokens": 742.7, "avg_reasoning_tokens": 398.5, "avg_cache_read_tokens": 58.5, "avg_cache_write_tokens": 2183.7, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 11.7, "median_query_seconds": 9.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 86.467, "latency": 23.395, "stderr": 1.031, "cost_per_test": 0.104874, "avg_input_tokens": 2244.2, "avg_output_tokens": 1552.8, "avg_cache_read_tokens": 8.7, "avg_cache_write_tokens": 1947.4, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 23.4, "median_query_seconds": 16.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 84.105, "latency": 22.32, "stderr": 1.102, "cost_per_test": 0.025774, "avg_input_tokens": 1454.9, "avg_output_tokens": 1905.4, "avg_reasoning_tokens": 1633.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 22.3, "median_query_seconds": 14.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 82.47, "latency": 30.789, "stderr": 1.146, "cost_per_test": 0.044563, "avg_input_tokens": 1454.9, "avg_output_tokens": 3471.1, "avg_reasoning_tokens": 3052.4, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 30.8, "median_query_seconds": 23.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 82.107, "latency": 259.071, "stderr": 1.184, "cost_per_test": 0.098415, "avg_input_tokens": 2249.2, "avg_output_tokens": 3427.8, "avg_cache_read_tokens": 169.6, "avg_cache_write_tokens": 1789.1, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 259.1, "median_query_seconds": 185.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 82.016, "latency": 10.668, "stderr": 1.162, "cost_per_test": 0.030686, "avg_input_tokens": 1454.9, "avg_output_tokens": 3800.5, "avg_reasoning_tokens": 3542.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 10.7, "median_query_seconds": 5.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 82.016, "latency": 394.222, "stderr": 1.175, "cost_per_test": 0.023265, "avg_input_tokens": 1455.1, "avg_output_tokens": 3651.4, "avg_reasoning_tokens": 3414.0, "avg_cache_read_tokens": 1035.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 394.2, "median_query_seconds": 307.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 81.926, "latency": 4.67, "stderr": 1.16, "cost_per_test": 0.011639, "avg_input_tokens": 1454.9, "avg_output_tokens": 1260.9, "avg_reasoning_tokens": 1058.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 3.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 81.926, "latency": 46.643, "stderr": 1.16, "cost_per_test": 0.113953, "avg_input_tokens": 2244.2, "avg_output_tokens": 4029.6, "avg_cache_read_tokens": 60.9, "avg_cache_write_tokens": 1894.9, "turn_metrics": { "tasks_measured": 1100, "avg_turns": 1.0, "avg_query_seconds": 46.6, "median_query_seconds": 19.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 81.835, "latency": 108.139, "stderr": 1.162, "cost_per_test": 0.022452, "avg_input_tokens": 1455.1, "avg_output_tokens": 3475.9, "avg_reasoning_tokens": 3304.0, "avg_cache_read_tokens": 875.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 108.1, "median_query_seconds": 43.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 80.381, "latency": 45.82, "stderr": 1.199, "cost_per_test": 0.028158, "avg_input_tokens": 1454.9, "avg_output_tokens": 2104.0, "avg_reasoning_tokens": 1834.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 45.8, "median_query_seconds": 33.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 80.109, "latency": 8.787, "stderr": 1.203, "cost_per_test": 0.014604, "avg_input_tokens": 1454.9, "avg_output_tokens": 1656.3, "avg_reasoning_tokens": 1419.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 8.8, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 79.927, "latency": 26.755, "stderr": 1.209, "cost_per_test": 0.111326, "avg_input_tokens": 1588.4, "avg_output_tokens": 1166.7, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 26.8, "median_query_seconds": 25.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 79.746, "latency": 60.609, "stderr": 1.506, "cost_per_test": 0.055171, "avg_input_tokens": 1577.4, "avg_output_tokens": 3362.6, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 60.6, "median_query_seconds": 27.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 79.473, "latency": 32.261, "stderr": 1.217, "cost_per_test": 0.017397, "avg_input_tokens": 1484.3, "avg_output_tokens": 1824.8, "avg_reasoning_tokens": 1514.2, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 32.3, "median_query_seconds": 29.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 79.11, "latency": 25.942, "stderr": 1.225, "cost_per_test": 0.053742, "avg_input_tokens": 1442.0, "avg_output_tokens": 1552.8, "avg_reasoning_tokens": 1483.7, "avg_cache_read_tokens": 11.6, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 25.9, "median_query_seconds": 11.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 78.928, "latency": 43.414, "stderr": 1.229, "cost_per_test": 0.003182, "avg_input_tokens": 1450.1, "avg_output_tokens": 2231.4, "avg_cache_read_tokens": 112.6, "avg_cache_write_tokens": 1335.5, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 43.4, "median_query_seconds": 27.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 78.747, "latency": 6.521, "stderr": 1.266, "cost_per_test": 0.015722, "avg_input_tokens": 2244.2, "avg_output_tokens": 491.2, "avg_cache_read_tokens": 17.6, "avg_cache_write_tokens": 2224.6, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 6.5, "median_query_seconds": 3.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 78.656, "latency": 66.38, "stderr": 1.342, "cost_per_test": 0.312744, "avg_input_tokens": 1589.4, "avg_output_tokens": 3852.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 66.4, "median_query_seconds": 42.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 78.474, "latency": 54.017, "stderr": 1.239, "cost_per_test": 0.027027, "avg_input_tokens": 1454.9, "avg_output_tokens": 8766.4, "avg_reasoning_tokens": 8412.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 54.0, "median_query_seconds": 54.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 78.292, "latency": 10.208, "stderr": 1.254, "cost_per_test": 0.054909, "avg_input_tokens": 1560.4, "avg_output_tokens": 420.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 10.2, "median_query_seconds": 10.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 77.475, "latency": 18.284, "stderr": 1.259, "cost_per_test": 0.009339, "avg_input_tokens": 1542.1, "avg_output_tokens": 363.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 18.3, "median_query_seconds": 18.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 77.293, "latency": 136.785, "stderr": 1.263, "cost_per_test": 0.035829, "avg_input_tokens": 1522.3, "avg_output_tokens": 5761.1, "avg_reasoning_tokens": 5599.8, "avg_cache_read_tokens": 1018.4, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 136.8, "median_query_seconds": 74.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k3": { "accuracy": 77.021, "latency": 52.421, "stderr": 1.268, "cost_per_test": 0.026007, "avg_input_tokens": 1538.3, "avg_output_tokens": 1675.3, "avg_reasoning_tokens": 1425.6, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 52.4, "median_query_seconds": 27.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.5": { "accuracy": 76.93, "latency": 68.6, "stderr": 1.27, "cost_per_test": 0.053088, "avg_input_tokens": 1441.4, "avg_output_tokens": 2713.1, "avg_reasoning_tokens": 2612.1, "avg_cache_read_tokens": 26.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 68.6, "median_query_seconds": 36.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 76.658, "latency": 63.765, "stderr": 1.28, "cost_per_test": 0.014682, "avg_input_tokens": 1443.5, "avg_output_tokens": 3293.6, "avg_reasoning_tokens": 2996.8, "avg_cache_read_tokens": 1017.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 63.8, "median_query_seconds": 42.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 76.476, "latency": 25.461, "stderr": 1.288, "cost_per_test": 0.012445, "avg_input_tokens": 1600.5, "avg_output_tokens": 2765.2, "avg_reasoning_tokens": 2530.5, "avg_cache_read_tokens": 1189.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 25.5, "median_query_seconds": 19.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 75.84, "latency": 33.183, "stderr": 1.29, "cost_per_test": 0.024716, "avg_input_tokens": 1441.4, "avg_output_tokens": 2342.6, "avg_reasoning_tokens": 2153.5, "avg_cache_read_tokens": 455.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 33.2, "median_query_seconds": 22.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 75.658, "latency": 49.942, "stderr": 1.293, "cost_per_test": 0.009164, "avg_input_tokens": 1484.3, "avg_output_tokens": 2807.4, "avg_reasoning_tokens": 2600.0, "turn_metrics": { "tasks_measured": 1100, "avg_turns": 1.0, "avg_query_seconds": 49.9, "median_query_seconds": 46.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 75.477, "latency": 45.188, "stderr": 1.456, "cost_per_test": 0.093825, "avg_input_tokens": 1441.4, "avg_output_tokens": 3062.5, "avg_reasoning_tokens": 2945.8, "avg_cache_read_tokens": 392.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 45.2, "median_query_seconds": 28.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 75.295, "latency": 54.392, "stderr": 1.3, "cost_per_test": 0.034743, "avg_input_tokens": 1441.4, "avg_output_tokens": 3294.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 54.4, "median_query_seconds": 46.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 75.023, "latency": 10.251, "stderr": 1.305, "cost_per_test": 0.001055, "avg_input_tokens": 1454.9, "avg_output_tokens": 2274.3, "avg_reasoning_tokens": 1984.3, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 10.3, "median_query_seconds": 7.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark": { "accuracy": 75.023, "latency": 47.886, "stderr": 1.305, "cost_per_test": 0.001906, "avg_input_tokens": 1498.6, "avg_output_tokens": 2807.9, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 47.9, "median_query_seconds": 42.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 74.932, "latency": 75.03, "stderr": 1.463, "cost_per_test": 0.063558, "avg_input_tokens": 1588.4, "avg_output_tokens": 3919.5, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 75.0, "median_query_seconds": 81.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 74.932, "latency": 118.275, "stderr": 1.312, "cost_per_test": 0.022229, "avg_input_tokens": 1524.8, "avg_output_tokens": 5482.7, "avg_reasoning_tokens": 5391.5, "avg_cache_read_tokens": 1172.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 118.3, "median_query_seconds": 49.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 74.569, "latency": 11.27, "stderr": 1.312, "cost_per_test": 0.050234, "avg_input_tokens": 1560.4, "avg_output_tokens": 357.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 11.3, "median_query_seconds": 11.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 74.387, "latency": 9.223, "stderr": 1.317, "cost_per_test": 0.011381, "avg_input_tokens": 1441.4, "avg_output_tokens": 711.0, "avg_reasoning_tokens": 641.9, "avg_cache_read_tokens": 18.6, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 9.2, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 74.387, "latency": 345.632, "stderr": 1.315, "cost_per_test": 0.013321, "avg_input_tokens": 1419.1, "avg_output_tokens": 5078.3, "avg_reasoning_tokens": 4766.3, "avg_cache_read_tokens": 1094.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 345.6, "median_query_seconds": 350.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 74.296, "latency": 272.45, "stderr": 1.323, "cost_per_test": 0.028894, "avg_input_tokens": 1460.3, "avg_output_tokens": 7079.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 958.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 272.4, "median_query_seconds": 191.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3": { "accuracy": 74.205, "latency": 119.076, "stderr": 1.319, "cost_per_test": 0.032863, "avg_input_tokens": 1465.1, "avg_output_tokens": 7159.1, "avg_reasoning_tokens": 6859.9, "avg_cache_read_tokens": 603.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 119.1, "median_query_seconds": 32.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 74.024, "latency": 535.644, "stderr": 1.345, "cost_per_test": 0.02826, "avg_input_tokens": 1511.8, "avg_output_tokens": 7010.0, "avg_reasoning_tokens": 6873.6, "avg_cache_read_tokens": 1171.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 535.6, "median_query_seconds": 598.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.1": { "accuracy": 74.0, "latency": 57.748, "stderr": 1.323, "cost_per_test": 0.010337, "avg_input_tokens": 1458.0, "avg_output_tokens": 3046.3, "avg_reasoning_tokens": 2694.9, "avg_cache_read_tokens": 1086.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1100, "avg_turns": 1.0, "avg_query_seconds": 57.7, "median_query_seconds": 36.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 73.933, "latency": 98.798, "stderr": 1.323, "cost_per_test": 0.011867, "avg_input_tokens": 1484.6, "avg_output_tokens": 4697.2, "avg_reasoning_tokens": 4273.4, "turn_metrics": { "tasks_measured": 1098, "avg_turns": 1.0, "avg_query_seconds": 98.8, "median_query_seconds": 89.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 73.569, "latency": 72.778, "stderr": 1.442, "cost_per_test": 0.073486, "avg_input_tokens": 1577.4, "avg_output_tokens": 4583.3, "avg_cache_write_tokens": 5.7, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 72.8, "median_query_seconds": 42.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 73.479, "latency": 72.826, "stderr": 1.35, "cost_per_test": 0.067295, "avg_input_tokens": 1468.3, "avg_output_tokens": 9209.5, "avg_reasoning_tokens": 4553.3, "avg_cache_read_tokens": 292.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 72.8, "median_query_seconds": 36.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 73.297, "latency": 20.864, "stderr": 1.333, "cost_per_test": 0.012843, "avg_input_tokens": 1441.4, "avg_output_tokens": 1245.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 20.9, "median_query_seconds": 14.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 73.025, "latency": 29.858, "stderr": 1.339, "cost_per_test": 0.005721, "avg_input_tokens": 1441.4, "avg_output_tokens": 4538.5, "avg_reasoning_tokens": 4456.5, "avg_cache_read_tokens": 75.2, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 29.9, "median_query_seconds": 11.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 72.934, "latency": 13.977, "stderr": 1.339, "cost_per_test": 0.016795, "avg_input_tokens": 1532.2, "avg_output_tokens": 2635.4, "avg_reasoning_tokens": 2419.2, "avg_cache_read_tokens": 1156.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 14.0, "median_query_seconds": 10.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 72.934, "latency": 33.366, "stderr": 1.339, "cost_per_test": null, "avg_input_tokens": 1484.0, "avg_output_tokens": 3259.1, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 33.4, "median_query_seconds": 18.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "zai/glm-5-thinking": { "accuracy": 72.843, "latency": 97.959, "stderr": 1.34, "cost_per_test": 0.012339, "avg_input_tokens": 1458.1, "avg_output_tokens": 3664.9, "avg_cache_read_tokens": 1044.8, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 98.0, "median_query_seconds": 79.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 72.48, "latency": 6.529, "stderr": 1.346, "cost_per_test": 0.005367, "avg_input_tokens": 1454.9, "avg_output_tokens": 1972.3, "avg_reasoning_tokens": 1734.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 6.5, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 72.207, "latency": 27.801, "stderr": 1.35, "cost_per_test": 0.003767, "avg_input_tokens": 1511.8, "avg_output_tokens": 2812.3, "avg_reasoning_tokens": 2740.9, "avg_cache_read_tokens": 1433.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 27.8, "median_query_seconds": 10.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 72.025, "latency": 72.715, "stderr": 1.353, "cost_per_test": 0.018016, "avg_input_tokens": 1465.1, "avg_output_tokens": 3840.0, "avg_reasoning_tokens": 3524.9, "avg_cache_read_tokens": 817.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 72.7, "median_query_seconds": 44.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 72.025, "latency": 105.186, "stderr": 1.353, "cost_per_test": 0.001281, "avg_input_tokens": 1433.2, "avg_output_tokens": 2944.7, "avg_reasoning_tokens": 2855.2, "avg_cache_read_tokens": 1373.7, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 105.2, "median_query_seconds": 84.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 71.571, "latency": 29.349, "stderr": 1.359, "cost_per_test": 0.137459, "avg_input_tokens": 1441.4, "avg_output_tokens": 1930.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 29.3, "median_query_seconds": 24.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 71.299, "latency": 65.411, "stderr": 1.363, "cost_per_test": 0.001518, "avg_input_tokens": 1465.1, "avg_output_tokens": 5784.2, "avg_cache_read_tokens": 635.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 65.4, "median_query_seconds": 15.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 71.208, "latency": 162.72, "stderr": 1.365, "cost_per_test": 0.087214, "avg_input_tokens": 1449.4, "avg_output_tokens": 9087.9, "avg_cache_read_tokens": 1092.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 162.7, "median_query_seconds": 99.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 71.117, "latency": 42.667, "stderr": 1.366, "cost_per_test": 0.001114, "avg_input_tokens": 1562.1, "avg_output_tokens": 1957.6, "avg_reasoning_tokens": 1805.0, "avg_cache_read_tokens": 1183.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 42.7, "median_query_seconds": 30.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 71.026, "latency": 18.956, "stderr": 1.367, "cost_per_test": 0.009806, "avg_input_tokens": 1453.1, "avg_output_tokens": 799.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 19.0, "median_query_seconds": 18.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-0709": { "accuracy": 71.026, "latency": 150.025, "stderr": 1.367, "cost_per_test": 0.058366, "avg_input_tokens": 1413.1, "avg_output_tokens": 3608.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 150.0, "median_query_seconds": 99.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 70.481, "latency": 141.377, "stderr": 1.375, "cost_per_test": 0.032196, "avg_input_tokens": 1465.8, "avg_output_tokens": 5072.8, "avg_reasoning_tokens": 4324.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1100, "avg_turns": 1.0, "avg_query_seconds": 141.4, "median_query_seconds": 130.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 70.391, "latency": 132.291, "stderr": 1.376, "cost_per_test": 0.013026, "avg_input_tokens": 1449.4, "avg_output_tokens": 10722.5, "avg_cache_read_tokens": 1147.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 132.3, "median_query_seconds": 88.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 70.3, "latency": 50.75, "stderr": 1.377, "cost_per_test": 0.009989, "avg_input_tokens": 1460.2, "avg_output_tokens": 3213.6, "avg_cache_read_tokens": 1037.5, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 50.8, "median_query_seconds": 42.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 70.118, "latency": 30.578, "stderr": 1.384, "cost_per_test": 0.006384, "avg_input_tokens": 1441.4, "avg_output_tokens": 3017.7, "avg_reasoning_tokens": 2921.0, "avg_cache_read_tokens": 51.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 30.6, "median_query_seconds": 19.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 69.846, "latency": 29.965, "stderr": 1.383, "cost_per_test": 0.001465, "avg_input_tokens": 1453.1, "avg_output_tokens": 1661.5, "avg_reasoning_tokens": 1249.9, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 30.0, "median_query_seconds": 12.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max": { "accuracy": 69.755, "latency": 0.0, "stderr": 1.384, "cost_per_test": 0.00685, "avg_input_tokens": 1463.7, "avg_output_tokens": 848.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 69.664, "latency": 11.73, "stderr": 1.385, "cost_per_test": 0.010381, "avg_input_tokens": 1560.4, "avg_output_tokens": 380.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 11.7, "median_query_seconds": 9.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 68.756, "latency": 11.802, "stderr": 1.397, "cost_per_test": 0.001468, "avg_input_tokens": 1453.1, "avg_output_tokens": 1654.8, "avg_reasoning_tokens": 1242.1, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 11.8, "median_query_seconds": 11.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 68.756, "latency": 30.476, "stderr": 1.476, "cost_per_test": 0.013571, "avg_input_tokens": 1431.8, "avg_output_tokens": 2155.2, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 30.5, "median_query_seconds": 22.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 68.483, "latency": 8.045, "stderr": 1.4, "cost_per_test": 0.001762, "avg_input_tokens": 1523.1, "avg_output_tokens": 2915.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 8.0, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 68.211, "latency": 38.848, "stderr": 1.415, "cost_per_test": 0.000331, "avg_input_tokens": 1432.8, "avg_output_tokens": 477.5, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1074.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 38.8, "median_query_seconds": 26.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 68.029, "latency": 44.386, "stderr": 1.406, "cost_per_test": 0.006265, "avg_input_tokens": 1463.7, "avg_output_tokens": 751.5, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 44.4, "median_query_seconds": 38.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 67.847, "latency": 54.787, "stderr": 1.408, "cost_per_test": 0.020667, "avg_input_tokens": 1526.3, "avg_output_tokens": 6634.6, "avg_reasoning_tokens": 6475.5, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 54.8, "median_query_seconds": 33.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 67.666, "latency": 6.734, "stderr": 1.41, "cost_per_test": 0.009054, "avg_input_tokens": 1560.4, "avg_output_tokens": 291.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 6.7, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 67.575, "latency": 32.762, "stderr": 1.411, "cost_per_test": 0.036944, "avg_input_tokens": 1588.4, "avg_output_tokens": 2145.3, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 32.8, "median_query_seconds": 23.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 66.939, "latency": 48.279, "stderr": 1.418, "cost_per_test": 0.002347, "avg_input_tokens": 1484.5, "avg_output_tokens": 5496.9, "avg_reasoning_tokens": 4995.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1099, "avg_turns": 1.0, "avg_query_seconds": 48.3, "median_query_seconds": 45.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 66.939, "latency": 74.224, "stderr": 1.507, "cost_per_test": 0.002031, "avg_input_tokens": 1706.6, "avg_output_tokens": 2005.8, "avg_reasoning_tokens": 1783.8, "avg_cache_read_tokens": 1046.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 74.2, "median_query_seconds": 48.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 66.939, "latency": 81.708, "stderr": 1.418, "cost_per_test": 0.069952, "avg_input_tokens": 1588.9, "avg_output_tokens": 4345.7, "turn_metrics": { "tasks_measured": 1085, "avg_turns": 1.0, "avg_query_seconds": 81.7, "median_query_seconds": 85.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 66.848, "latency": 6.963, "stderr": 1.419, "cost_per_test": 0.010008, "avg_input_tokens": 1560.4, "avg_output_tokens": 355.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 7.0, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 66.757, "latency": 37.066, "stderr": 1.42, "cost_per_test": 0.000908, "avg_input_tokens": 1702.7, "avg_output_tokens": 2898.1, "avg_reasoning_tokens": 2782.7, "avg_cache_read_tokens": 1034.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 37.1, "median_query_seconds": 32.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 66.485, "latency": 57.733, "stderr": 1.429, "cost_per_test": 0.005721, "avg_input_tokens": 1588.1, "avg_output_tokens": 2213.5, "avg_cache_read_tokens": 1133.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 57.7, "median_query_seconds": 36.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 65.577, "latency": 4.849, "stderr": 1.432, "cost_per_test": 0.006256, "avg_input_tokens": 1442.4, "avg_output_tokens": 421.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 4.8, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.5": { "accuracy": 65.123, "latency": 98.1, "stderr": 1.436, "cost_per_test": 0.008511, "avg_input_tokens": 1458.1, "avg_output_tokens": 3471.1, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 98.1, "median_query_seconds": 93.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-4.7": { "accuracy": 64.759, "latency": 121.12, "stderr": 1.44, "cost_per_test": 0.00951, "avg_input_tokens": 1458.1, "avg_output_tokens": 4126.5, "avg_cache_read_tokens": 904.1, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 121.1, "median_query_seconds": 62.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 64.124, "latency": 17.7, "stderr": 1.446, "cost_per_test": null, "avg_input_tokens": 1480.9, "avg_output_tokens": 808.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 17.7, "median_query_seconds": 17.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 63.669, "latency": 10.826, "stderr": 1.449, "cost_per_test": 0.001425, "avg_input_tokens": 1413.1, "avg_output_tokens": 2003.1, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 10.8, "median_query_seconds": 9.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 63.306, "latency": 145.75, "stderr": 1.466, "cost_per_test": 0.003132, "avg_input_tokens": 1460.7, "avg_output_tokens": 4890.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 145.7, "median_query_seconds": 84.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 62.761, "latency": 22.692, "stderr": 1.457, "cost_per_test": 0.006819, "avg_input_tokens": 1441.4, "avg_output_tokens": 3229.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 22.7, "median_query_seconds": 20.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 62.67, "latency": 21.29, "stderr": 1.458, "cost_per_test": 0.00311, "avg_input_tokens": 1469.3, "avg_output_tokens": 546.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 21.3, "median_query_seconds": 17.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 62.216, "latency": 43.716, "stderr": 1.507, "cost_per_test": 0.025813, "avg_input_tokens": 1588.4, "avg_output_tokens": 4844.9, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 43.7, "median_query_seconds": 44.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 61.399, "latency": 12.946, "stderr": 1.467, "cost_per_test": 0.006345, "avg_input_tokens": 1441.4, "avg_output_tokens": 1081.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 12.9, "median_query_seconds": 11.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 60.854, "latency": 6.139, "stderr": 1.471, "cost_per_test": 0.000978, "avg_input_tokens": 1413.1, "avg_output_tokens": 1108.5, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 6.1, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 60.763, "latency": 3.779, "stderr": 1.472, "cost_per_test": 0.000633, "avg_input_tokens": 1446.5, "avg_output_tokens": 357.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 3.8, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-4.6": { "accuracy": 60.763, "latency": 47.136, "stderr": 1.472, "cost_per_test": 0.00724, "avg_input_tokens": 1459.8, "avg_output_tokens": 2892.9, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 47.1, "median_query_seconds": 35.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 60.672, "latency": 5.07, "stderr": 1.472, "cost_per_test": 0.000399, "avg_input_tokens": 1453.1, "avg_output_tokens": 634.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 5.1, "median_query_seconds": 1.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 60.581, "latency": 3.583, "stderr": 1.473, "cost_per_test": 0.000563, "avg_input_tokens": 1453.1, "avg_output_tokens": 1043.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 3.6, "median_query_seconds": 3.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 60.49, "latency": 64.611, "stderr": 1.476, "cost_per_test": 0.004826, "avg_input_tokens": 1433.1, "avg_output_tokens": 3601.8, "avg_cache_read_tokens": 96.2, "avg_cache_write_tokens": 1336.9, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 64.6, "median_query_seconds": 36.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-3": { "accuracy": 60.218, "latency": 11.861, "stderr": 1.475, "cost_per_test": 0.019033, "avg_input_tokens": 1412.1, "avg_output_tokens": 986.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 11.9, "median_query_seconds": 11.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 59.582, "latency": 2.596, "stderr": 1.479, "cost_per_test": 0.000217, "avg_input_tokens": 1574.1, "avg_output_tokens": 162.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1190.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 2.6, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-2-1212": { "accuracy": 59.401, "latency": 5.726, "stderr": 1.48, "cost_per_test": 0.00557, "avg_input_tokens": 1439.4, "avg_output_tokens": 269.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 5.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 59.401, "latency": 71.288, "stderr": 1.49, "cost_per_test": 0.002407, "avg_input_tokens": 1463.7, "avg_output_tokens": 2368.8, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 71.3, "median_query_seconds": 46.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 59.219, "latency": 66.126, "stderr": 1.482, "cost_per_test": 0.006817, "avg_input_tokens": 1453.1, "avg_output_tokens": 2630.3, "avg_cache_read_tokens": 116.6, "avg_cache_write_tokens": 1336.5, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 66.1, "median_query_seconds": 53.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 59.037, "latency": 10.155, "stderr": 1.482, "cost_per_test": 0.000305, "avg_input_tokens": 1453.1, "avg_output_tokens": 2011.4, "avg_reasoning_tokens": 1612.9, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 10.2, "median_query_seconds": 9.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 59.037, "latency": 69.15, "stderr": 1.483, "cost_per_test": 0.002942, "avg_input_tokens": 1509.4, "avg_output_tokens": 4526.6, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 69.1, "median_query_seconds": 64.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 58.856, "latency": 7.031, "stderr": 1.493, "cost_per_test": 0.000369, "avg_input_tokens": 1441.4, "avg_output_tokens": 741.8, "avg_reasoning_tokens": 610.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 7.0, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 58.492, "latency": 17.187, "stderr": 1.489, "cost_per_test": 0.000711, "avg_input_tokens": 1476.2, "avg_output_tokens": 3010.0, "avg_cache_read_tokens": 1026.6, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 17.2, "median_query_seconds": 9.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 57.856, "latency": 1.884, "stderr": 1.488, "cost_per_test": 0.001188, "avg_input_tokens": 1533.1, "avg_output_tokens": 1763.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 1.9, "median_query_seconds": 1.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 56.857, "latency": 72.657, "stderr": 1.493, "cost_per_test": 0.053547, "avg_input_tokens": 1483.0, "avg_output_tokens": 6842.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1074, "avg_turns": 1.0, "avg_query_seconds": 72.7, "median_query_seconds": 68.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 56.494, "latency": 37.236, "stderr": 1.494, "cost_per_test": 0.018636, "avg_input_tokens": 1471.0, "avg_output_tokens": 3138.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 37.2, "median_query_seconds": 31.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 56.403, "latency": 17.014, "stderr": 1.306, "cost_per_test": 0.001604, "avg_input_tokens": 1471.0, "avg_output_tokens": 580.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 17.0, "median_query_seconds": 16.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 56.403, "latency": 29.997, "stderr": 1.494, "cost_per_test": 0.001441, "avg_input_tokens": 1431.8, "avg_output_tokens": 169.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 30.0, "median_query_seconds": 25.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 56.131, "latency": 8.064, "stderr": 1.496, "cost_per_test": 0.00696, "avg_input_tokens": 1442.4, "avg_output_tokens": 335.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 8.1, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 55.223, "latency": 24.625, "stderr": 1.499, "cost_per_test": null, "avg_input_tokens": 1480.9, "avg_output_tokens": 1167.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 24.6, "median_query_seconds": 21.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 54.223, "latency": 2.483, "stderr": 1.501, "cost_per_test": 0.002383, "avg_input_tokens": 1453.1, "avg_output_tokens": 113.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 2.5, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 53.769, "latency": 3.66, "stderr": 1.503, "cost_per_test": 0.001316, "avg_input_tokens": 1442.4, "avg_output_tokens": 461.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 3.7, "median_query_seconds": 3.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 52.77, "latency": 6.141, "stderr": 1.505, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 52.407, "latency": 34.147, "stderr": 1.461, "cost_per_test": 0.000428, "avg_input_tokens": 1548.2, "avg_output_tokens": 1665.6, "avg_reasoning_tokens": 1360.6, "avg_cache_read_tokens": 1206.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 34.1, "median_query_seconds": 27.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-m.1": { "accuracy": 52.044, "latency": 94.539, "stderr": 1.507, "cost_per_test": 0.00127, "avg_input_tokens": 1511.2, "avg_output_tokens": 2711.3, "avg_reasoning_tokens": 2408.0, "avg_cache_read_tokens": 1170.6, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 94.5, "median_query_seconds": 72.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 51.771, "latency": 4.355, "stderr": 1.506, "cost_per_test": 0.001661, "avg_input_tokens": 1487.9, "avg_output_tokens": 400.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 4.4, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 51.408, "latency": 8.478, "stderr": 1.506, "cost_per_test": 0.008141, "avg_input_tokens": 1457.3, "avg_output_tokens": 449.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 8.5, "median_query_seconds": 7.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 51.045, "latency": 14.726, "stderr": 1.507, "cost_per_test": 0.001382, "avg_input_tokens": 1431.8, "avg_output_tokens": 103.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 14.7, "median_query_seconds": 12.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 50.772, "latency": 31.103, "stderr": 1.507, "cost_per_test": 0.018975, "avg_input_tokens": 1441.4, "avg_output_tokens": 3952.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 31.1, "median_query_seconds": 25.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 50.5, "latency": 32.578, "stderr": 1.507, "cost_per_test": 0.003021, "avg_input_tokens": 1441.4, "avg_output_tokens": 7372.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 32.6, "median_query_seconds": 30.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 49.046, "latency": 4.608, "stderr": 1.507, "cost_per_test": 0.004517, "avg_input_tokens": 1619.3, "avg_output_tokens": 213.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 4.6, "median_query_seconds": 4.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 48.683, "latency": 13.023, "stderr": 1.481, "cost_per_test": 0.008599, "avg_input_tokens": 1442.4, "avg_output_tokens": 499.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 13.0, "median_query_seconds": 12.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 47.048, "latency": 5.019, "stderr": 1.504, "cost_per_test": 0.00053, "avg_input_tokens": 1446.5, "avg_output_tokens": 457.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 5.0, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 45.504, "latency": 2.897, "stderr": 1.501, "cost_per_test": 0.000181, "avg_input_tokens": 1471.0, "avg_output_tokens": 235.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 2.9, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 44.596, "latency": 73.861, "stderr": 1.484, "cost_per_test": 0.001907, "avg_input_tokens": 1509.4, "avg_output_tokens": 6003.3, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 73.9, "median_query_seconds": 66.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 44.142, "latency": 6.261, "stderr": 1.496, "cost_per_test": 0.002355, "avg_input_tokens": 1560.4, "avg_output_tokens": 276.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 6.3, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 42.598, "latency": 1.323, "stderr": 1.49, "cost_per_test": 0.000161, "avg_input_tokens": 1453.1, "avg_output_tokens": 172.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 1.3, "median_query_seconds": 1.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 40.872, "latency": 3.275, "stderr": 1.482, "cost_per_test": 0.000464, "avg_input_tokens": 1471.0, "avg_output_tokens": 283.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 3.3, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 37.693, "latency": 3.593, "stderr": 1.461, "cost_per_test": 0.000363, "avg_input_tokens": 1442.4, "avg_output_tokens": 244.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 3.6, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 37.511, "latency": 2.468, "stderr": 1.456, "cost_per_test": 0.000364, "avg_input_tokens": 1442.4, "avg_output_tokens": 548.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 2.5, "median_query_seconds": 2.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "cohere/command-r-plus": { "accuracy": 36.058, "latency": 3.06, "stderr": 1.446, "cost_per_test": 0.005053, "avg_input_tokens": 1457.3, "avg_output_tokens": 141.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 3.1, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 35.695, "latency": 7.047, "stderr": 1.443, "cost_per_test": 0.004165, "avg_input_tokens": 1524.6, "avg_output_tokens": 139.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 7.0, "median_query_seconds": 5.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 28.247, "latency": 17.029, "stderr": 1.357, "cost_per_test": 0.003685, "avg_input_tokens": 1471.0, "avg_output_tokens": 1966.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 17.0, "median_query_seconds": 12.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 27.157, "latency": 1.471, "stderr": 1.34, "cost_per_test": 0.000355, "avg_input_tokens": 1524.6, "avg_output_tokens": 125.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1101, "avg_turns": 1.0, "avg_query_seconds": 1.5, "median_query_seconds": 1.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "math": { "anthropic/claude-fable-5": { "accuracy": 97.557, "latency": 21.371, "stderr": 0.42, "cost_per_test": 0.07686, "avg_input_tokens": 2290.4, "avg_output_tokens": 964.7, "avg_cache_write_tokens": 2288.4, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 21.4, "median_query_seconds": 9.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 97.335, "latency": 11.096, "stderr": 0.438, "cost_per_test": 0.065167, "avg_input_tokens": 2292.4, "avg_output_tokens": 730.4, "avg_cache_write_tokens": 2288.4, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 11.1, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 97.113, "latency": 7.774, "stderr": 0.461, "cost_per_test": 0.028227, "avg_input_tokens": 2290.4, "avg_output_tokens": 556.6, "avg_reasoning_tokens": 294.2, "avg_cache_write_tokens": 2288.4, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 7.8, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 96.299, "latency": 50.407, "stderr": 0.533, "cost_per_test": 0.023637, "avg_input_tokens": 1794.7, "avg_output_tokens": 1670.6, "avg_reasoning_tokens": 1323.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 50.4, "median_query_seconds": 33.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 96.151, "latency": 5.444, "stderr": 0.523, "cost_per_test": 0.018302, "avg_input_tokens": 1794.3, "avg_output_tokens": 2081.4, "avg_reasoning_tokens": 1777.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 5.4, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 96.003, "latency": 3.581, "stderr": 0.538, "cost_per_test": 0.011336, "avg_input_tokens": 1794.3, "avg_output_tokens": 1152.7, "avg_reasoning_tokens": 874.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 3.6, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 96.003, "latency": 6.718, "stderr": 0.533, "cost_per_test": 0.013897, "avg_input_tokens": 1794.3, "avg_output_tokens": 1494.0, "avg_reasoning_tokens": 1185.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 6.7, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 96.003, "latency": 18.126, "stderr": 0.547, "cost_per_test": 0.009214, "avg_input_tokens": 1810.1, "avg_output_tokens": 2044.4, "avg_reasoning_tokens": 1836.2, "avg_cache_read_tokens": 1573.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 18.1, "median_query_seconds": 12.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 96.003, "latency": 24.058, "stderr": 0.533, "cost_per_test": 0.03927, "avg_input_tokens": 1796.0, "avg_output_tokens": 2973.1, "avg_reasoning_tokens": 2459.5, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 24.1, "median_query_seconds": 16.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 96.003, "latency": 24.464, "stderr": 0.533, "cost_per_test": 0.026993, "avg_input_tokens": 1796.0, "avg_output_tokens": 1950.1, "avg_reasoning_tokens": 1556.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 24.5, "median_query_seconds": 14.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 96.003, "latency": 29.213, "stderr": 0.533, "cost_per_test": 0.019017, "avg_input_tokens": 1793.7, "avg_output_tokens": 1937.7, "avg_reasoning_tokens": 1516.6, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 29.2, "median_query_seconds": 22.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 96.003, "latency": 31.834, "stderr": 0.533, "cost_per_test": 0.008477, "avg_input_tokens": 1653.1, "avg_output_tokens": 1879.2, "avg_reasoning_tokens": 1651.9, "avg_cache_read_tokens": 1432.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 31.8, "median_query_seconds": 16.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 95.929, "latency": 36.687, "stderr": 0.685, "cost_per_test": 0.068593, "avg_input_tokens": 2290.4, "avg_output_tokens": 4000.4, "avg_cache_write_tokens": 2288.4, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 36.7, "median_query_seconds": 7.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 95.929, "latency": 39.766, "stderr": 0.538, "cost_per_test": 0.013407, "avg_input_tokens": 1744.2, "avg_output_tokens": 1993.6, "avg_reasoning_tokens": 1812.3, "avg_cache_read_tokens": 1361.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 39.8, "median_query_seconds": 14.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 95.929, "latency": 41.961, "stderr": 1.024, "cost_per_test": 0.051479, "avg_input_tokens": 1968.2, "avg_output_tokens": 3038.3, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 42.0, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 95.929, "latency": 340.418, "stderr": 0.551, "cost_per_test": 0.014781, "avg_input_tokens": 1744.2, "avg_output_tokens": 2253.6, "avg_reasoning_tokens": 1996.1, "avg_cache_read_tokens": 1486.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 340.4, "median_query_seconds": 355.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 95.855, "latency": 18.449, "stderr": 0.542, "cost_per_test": 0.055314, "avg_input_tokens": 2290.4, "avg_output_tokens": 1640.0, "avg_cache_write_tokens": 2288.4, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 18.4, "median_query_seconds": 7.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 95.855, "latency": 40.377, "stderr": 0.542, "cost_per_test": 0.012165, "avg_input_tokens": 1713.3, "avg_output_tokens": 2535.2, "avg_reasoning_tokens": 2176.6, "avg_cache_read_tokens": 1217.7, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 40.4, "median_query_seconds": 19.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 95.855, "latency": 79.094, "stderr": 0.542, "cost_per_test": 0.042614, "avg_input_tokens": 2295.4, "avg_output_tokens": 2083.0, "avg_cache_write_tokens": 2289.4, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 79.1, "median_query_seconds": 62.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 95.707, "latency": 36.885, "stderr": 0.551, "cost_per_test": 0.007629, "avg_input_tokens": 1666.2, "avg_output_tokens": 2457.9, "avg_cache_read_tokens": 1478.2, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 36.9, "median_query_seconds": 24.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 95.707, "latency": 39.099, "stderr": 0.551, "cost_per_test": 0.022316, "avg_input_tokens": 1794.3, "avg_output_tokens": 7139.6, "avg_reasoning_tokens": 6694.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 39.1, "median_query_seconds": 22.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 95.633, "latency": 33.852, "stderr": 0.556, "cost_per_test": 0.016185, "avg_input_tokens": 1744.2, "avg_output_tokens": 1012.7, "avg_reasoning_tokens": 828.8, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 33.9, "median_query_seconds": 14.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 95.633, "latency": 140.155, "stderr": 0.556, "cost_per_test": 0.014751, "avg_input_tokens": 1666.2, "avg_output_tokens": 3570.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1397.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 140.2, "median_query_seconds": 54.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 95.559, "latency": 30.914, "stderr": 0.56, "cost_per_test": null, "avg_input_tokens": 1757.8, "avg_output_tokens": 2888.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 30.9, "median_query_seconds": 9.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 95.559, "latency": 78.179, "stderr": 0.56, "cost_per_test": 0.009015, "avg_input_tokens": 1793.7, "avg_output_tokens": 3457.3, "avg_reasoning_tokens": 2938.2, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 78.2, "median_query_seconds": 60.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 95.559, "latency": 89.333, "stderr": 0.56, "cost_per_test": 0.024008, "avg_input_tokens": 1778.3, "avg_output_tokens": 3645.7, "avg_reasoning_tokens": 3056.2, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 89.3, "median_query_seconds": 47.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 95.485, "latency": 41.346, "stderr": 0.565, "cost_per_test": 0.013284, "avg_input_tokens": 1831.7, "avg_output_tokens": 1899.9, "avg_reasoning_tokens": 1680.6, "avg_cache_read_tokens": 1016.4, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 41.3, "median_query_seconds": 12.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 95.485, "latency": 50.573, "stderr": 0.565, "cost_per_test": 0.00986, "avg_input_tokens": 1793.7, "avg_output_tokens": 2987.6, "avg_reasoning_tokens": 2641.2, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 50.6, "median_query_seconds": 42.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.1": { "accuracy": 95.485, "latency": 62.824, "stderr": 0.565, "cost_per_test": 0.01127, "avg_input_tokens": 1706.3, "avg_output_tokens": 3368.7, "avg_reasoning_tokens": 3055.8, "avg_cache_read_tokens": 1520.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 62.8, "median_query_seconds": 22.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 95.337, "latency": 474.812, "stderr": 0.578, "cost_per_test": 0.022908, "avg_input_tokens": 1743.6, "avg_output_tokens": 5593.9, "avg_reasoning_tokens": 5396.7, "avg_cache_read_tokens": 1210.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 474.8, "median_query_seconds": 550.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 95.263, "latency": 9.448, "stderr": 0.578, "cost_per_test": 0.021345, "avg_input_tokens": 1734.7, "avg_output_tokens": 605.1, "avg_reasoning_tokens": 498.9, "avg_cache_read_tokens": 1218.4, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 9.4, "median_query_seconds": 4.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 95.263, "latency": 43.855, "stderr": 0.578, "cost_per_test": 0.00394, "avg_input_tokens": 1713.6, "avg_output_tokens": 3178.9, "avg_cache_read_tokens": 1620.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 43.9, "median_query_seconds": 10.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5.3": { "accuracy": 95.189, "latency": 34.75, "stderr": 0.582, "cost_per_test": 0.01228, "avg_input_tokens": 1713.3, "avg_output_tokens": 2526.8, "avg_reasoning_tokens": 2248.7, "avg_cache_read_tokens": 1085.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 34.8, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 95.189, "latency": 35.793, "stderr": 0.582, "cost_per_test": 0.000859, "avg_input_tokens": 1713.3, "avg_output_tokens": 3223.2, "avg_cache_read_tokens": 1246.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 35.8, "median_query_seconds": 6.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 95.115, "latency": 22.816, "stderr": 0.586, "cost_per_test": 0.013096, "avg_input_tokens": 1760.4, "avg_output_tokens": 2096.1, "avg_reasoning_tokens": 1846.9, "avg_cache_read_tokens": 1667.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 22.8, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 95.115, "latency": 39.306, "stderr": 0.603, "cost_per_test": 0.008654, "avg_input_tokens": 1756.6, "avg_output_tokens": 2094.9, "avg_reasoning_tokens": 1955.7, "avg_cache_read_tokens": 1536.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 39.3, "median_query_seconds": 9.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3-max": { "accuracy": 95.041, "latency": 0.0, "stderr": 0.591, "cost_per_test": 0.007666, "avg_input_tokens": 1776.3, "avg_output_tokens": 922.5, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 95.041, "latency": 26.339, "stderr": 0.599, "cost_per_test": 0.031514, "avg_input_tokens": 1721.1, "avg_output_tokens": 4119.1, "avg_reasoning_tokens": 1971.5, "avg_cache_read_tokens": 993.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 26.3, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 95.041, "latency": 56.02, "stderr": 0.591, "cost_per_test": 0.001713, "avg_input_tokens": 1708.0, "avg_output_tokens": 2432.1, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 56.0, "median_query_seconds": 47.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 95.041, "latency": 73.223, "stderr": 0.591, "cost_per_test": 0.000995, "avg_input_tokens": 1664.5, "avg_output_tokens": 2227.3, "avg_reasoning_tokens": 2095.1, "avg_cache_read_tokens": 1606.4, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 73.2, "median_query_seconds": 36.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 94.967, "latency": 37.162, "stderr": 1.123, "cost_per_test": 0.048284, "avg_input_tokens": 1979.2, "avg_output_tokens": 2823.1, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 37.2, "median_query_seconds": 17.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 94.893, "latency": 27.458, "stderr": 0.599, "cost_per_test": 0.001691, "avg_input_tokens": 1793.7, "avg_output_tokens": 3780.2, "avg_reasoning_tokens": 3231.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 27.5, "median_query_seconds": 21.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 94.893, "latency": 46.611, "stderr": 0.599, "cost_per_test": 0.023878, "avg_input_tokens": 1713.6, "avg_output_tokens": 2376.9, "avg_cache_read_tokens": 1596.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 46.6, "median_query_seconds": 12.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 94.819, "latency": 10.308, "stderr": 0.603, "cost_per_test": 0.001361, "avg_input_tokens": 1794.3, "avg_output_tokens": 2954.6, "avg_reasoning_tokens": 2522.1, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 10.3, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5-thinking": { "accuracy": 94.819, "latency": 79.691, "stderr": 0.603, "cost_per_test": 0.012724, "avg_input_tokens": 1706.3, "avg_output_tokens": 3814.7, "avg_cache_read_tokens": 1486.4, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 79.7, "median_query_seconds": 44.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 94.819, "latency": 97.073, "stderr": 0.603, "cost_per_test": 0.008576, "avg_input_tokens": 1835.7, "avg_output_tokens": 2552.8, "avg_reasoning_tokens": 2335.1, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 97.1, "median_query_seconds": 29.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.5": { "accuracy": 94.745, "latency": 33.027, "stderr": 0.607, "cost_per_test": 0.017893, "avg_input_tokens": 1716.3, "avg_output_tokens": 896.5, "avg_reasoning_tokens": 728.2, "avg_cache_read_tokens": 1256.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 33.0, "median_query_seconds": 30.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 94.745, "latency": 36.702, "stderr": 0.607, "cost_per_test": 0.023739, "avg_input_tokens": 1705.6, "avg_output_tokens": 2160.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 36.7, "median_query_seconds": 14.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 94.671, "latency": 16.611, "stderr": 0.805, "cost_per_test": 0.037007, "avg_input_tokens": 1705.6, "avg_output_tokens": 1185.6, "avg_reasoning_tokens": 989.5, "avg_cache_read_tokens": 462.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 16.6, "median_query_seconds": 9.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 94.597, "latency": 19.462, "stderr": 0.619, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 94.597, "latency": 34.289, "stderr": 0.675, "cost_per_test": 0.004775, "avg_input_tokens": 1814.5, "avg_output_tokens": 1865.4, "avg_cache_read_tokens": 1646.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 34.3, "median_query_seconds": 12.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 94.523, "latency": 15.983, "stderr": 0.671, "cost_per_test": 0.00061, "avg_input_tokens": 1771.4, "avg_output_tokens": 2584.3, "avg_cache_read_tokens": 1520.7, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 16.0, "median_query_seconds": 5.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 94.449, "latency": 4.909, "stderr": 0.623, "cost_per_test": 0.005287, "avg_input_tokens": 1794.3, "avg_output_tokens": 1899.6, "avg_reasoning_tokens": 1557.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 4.9, "median_query_seconds": 3.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 94.449, "latency": 15.14, "stderr": 0.638, "cost_per_test": 0.00289, "avg_input_tokens": 1705.6, "avg_output_tokens": 1374.9, "avg_reasoning_tokens": 1194.0, "avg_cache_read_tokens": 1271.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 15.1, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 94.301, "latency": 12.822, "stderr": 0.631, "cost_per_test": 0.004542, "avg_input_tokens": 1705.6, "avg_output_tokens": 2058.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 12.8, "median_query_seconds": 6.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 94.301, "latency": 20.251, "stderr": 0.635, "cost_per_test": 0.019754, "avg_input_tokens": 1705.6, "avg_output_tokens": 1820.6, "avg_reasoning_tokens": 1544.6, "avg_cache_read_tokens": 519.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 20.3, "median_query_seconds": 11.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 94.301, "latency": 44.587, "stderr": 0.631, "cost_per_test": 0.007185, "avg_input_tokens": 1776.3, "avg_output_tokens": 842.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 44.6, "median_query_seconds": 25.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 94.226, "latency": 36.869, "stderr": 0.653, "cost_per_test": 0.180541, "avg_input_tokens": 1979.2, "avg_output_tokens": 2011.4, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 36.9, "median_query_seconds": 21.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 94.152, "latency": 5.296, "stderr": 0.638, "cost_per_test": 0.005209, "avg_input_tokens": 1705.6, "avg_output_tokens": 341.2, "avg_reasoning_tokens": 237.7, "avg_cache_read_tokens": 1276.2, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 5.3, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 94.152, "latency": 34.451, "stderr": 1.088, "cost_per_test": 0.228935, "avg_input_tokens": 1980.2, "avg_output_tokens": 2656.4, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 34.5, "median_query_seconds": 14.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 94.078, "latency": 45.334, "stderr": 1.353, "cost_per_test": 0.065662, "avg_input_tokens": 1968.2, "avg_output_tokens": 3972.4, "avg_cache_write_tokens": 228.1, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 45.3, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 94.004, "latency": 12.347, "stderr": 0.646, "cost_per_test": 0.002039, "avg_input_tokens": 1743.6, "avg_output_tokens": 1457.7, "avg_reasoning_tokens": 1339.6, "avg_cache_read_tokens": 1531.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 12.3, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 94.004, "latency": 20.683, "stderr": 0.646, "cost_per_test": 0.01422, "avg_input_tokens": 1705.6, "avg_output_tokens": 1351.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 20.7, "median_query_seconds": 8.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 94.004, "latency": 27.882, "stderr": 0.653, "cost_per_test": 0.000807, "avg_input_tokens": 2015.3, "avg_output_tokens": 2493.8, "avg_reasoning_tokens": 2295.4, "avg_cache_read_tokens": 1266.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 27.9, "median_query_seconds": 12.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 93.708, "latency": 10.711, "stderr": 0.661, "cost_per_test": 0.001851, "avg_input_tokens": 1793.4, "avg_output_tokens": 2121.9, "avg_reasoning_tokens": 1596.9, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 10.7, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 93.708, "latency": 10.815, "stderr": 0.661, "cost_per_test": 0.064139, "avg_input_tokens": 1951.2, "avg_output_tokens": 465.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 10.8, "median_query_seconds": 9.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.6": { "accuracy": 93.634, "latency": 51.314, "stderr": 0.699, "cost_per_test": 0.00669, "avg_input_tokens": 1706.3, "avg_output_tokens": 2575.3, "avg_cache_read_tokens": 1375.2, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 51.3, "median_query_seconds": 23.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 93.634, "latency": 71.077, "stderr": 0.86, "cost_per_test": 0.002403, "avg_input_tokens": 2019.2, "avg_output_tokens": 2404.4, "avg_reasoning_tokens": 2198.3, "avg_cache_read_tokens": 1309.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 71.1, "median_query_seconds": 30.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 93.412, "latency": 10.848, "stderr": 0.728, "cost_per_test": 0.001959, "avg_input_tokens": 1705.6, "avg_output_tokens": 1348.2, "avg_reasoning_tokens": 1223.8, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 10.8, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 93.338, "latency": 17.095, "stderr": 0.685, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 93.264, "latency": 20.432, "stderr": 0.682, "cost_per_test": 0.011498, "avg_input_tokens": 1705.6, "avg_output_tokens": 2186.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 20.4, "median_query_seconds": 10.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 93.116, "latency": 7.385, "stderr": 0.689, "cost_per_test": 0.00183, "avg_input_tokens": 1793.4, "avg_output_tokens": 2117.9, "avg_reasoning_tokens": 1601.3, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 7.4, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 93.042, "latency": 64.992, "stderr": 0.696, "cost_per_test": 0.064348, "avg_input_tokens": 1979.3, "avg_output_tokens": 3894.0, "turn_metrics": { "tasks_measured": 1346, "avg_turns": 1.0, "avg_query_seconds": 65.0, "median_query_seconds": 34.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 92.968, "latency": 26.179, "stderr": 1.043, "cost_per_test": 0.012333, "avg_input_tokens": 1663.6, "avg_output_tokens": 1868.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 26.2, "median_query_seconds": 15.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-4.7": { "accuracy": 92.968, "latency": 96.951, "stderr": 0.696, "cost_per_test": 0.008694, "avg_input_tokens": 1706.2, "avg_output_tokens": 3797.9, "avg_cache_read_tokens": 1306.3, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 97.0, "median_query_seconds": 37.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 92.82, "latency": 18.394, "stderr": 0.702, "cost_per_test": 0.001493, "avg_input_tokens": 1705.6, "avg_output_tokens": 3518.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 18.4, "median_query_seconds": 12.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 92.746, "latency": 55.213, "stderr": 0.706, "cost_per_test": 0.004308, "avg_input_tokens": 1676.5, "avg_output_tokens": 3075.8, "avg_cache_read_tokens": 33.4, "avg_cache_write_tokens": 1641.1, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 55.2, "median_query_seconds": 24.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 92.672, "latency": 49.349, "stderr": 0.712, "cost_per_test": 0.004255, "avg_input_tokens": 1659.5, "avg_output_tokens": 3031.6, "avg_cache_read_tokens": 16.0, "avg_cache_write_tokens": 1643.5, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 49.3, "median_query_seconds": 16.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 92.598, "latency": 42.988, "stderr": 0.719, "cost_per_test": 0.005288, "avg_input_tokens": 1679.5, "avg_output_tokens": 1946.2, "avg_cache_read_tokens": 36.0, "avg_cache_write_tokens": 1643.5, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 43.0, "median_query_seconds": 20.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 92.302, "latency": 7.848, "stderr": 1.063, "cost_per_test": 0.065281, "avg_input_tokens": 1951.2, "avg_output_tokens": 480.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 7.8, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 92.302, "latency": 9.219, "stderr": 0.725, "cost_per_test": 0.0055, "avg_input_tokens": 1705.6, "avg_output_tokens": 823.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 9.2, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 91.932, "latency": 24.494, "stderr": 0.744, "cost_per_test": 0.12593, "avg_input_tokens": 1705.6, "avg_output_tokens": 1672.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 24.5, "median_query_seconds": 14.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 91.71, "latency": 28.831, "stderr": 0.889, "cost_per_test": 0.000321, "avg_input_tokens": 1664.5, "avg_output_tokens": 574.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1529.8, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 28.8, "median_query_seconds": 25.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 91.636, "latency": 76.794, "stderr": 0.92, "cost_per_test": 0.002082, "avg_input_tokens": 1666.0, "avg_output_tokens": 2532.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 76.8, "median_query_seconds": 31.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 91.562, "latency": 48.296, "stderr": 0.852, "cost_per_test": 0.002828, "avg_input_tokens": 1776.3, "avg_output_tokens": 2770.1, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 48.3, "median_query_seconds": 25.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 91.414, "latency": 40.166, "stderr": 0.762, "cost_per_test": 0.05626, "avg_input_tokens": 1979.2, "avg_output_tokens": 3354.8, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 40.2, "median_query_seconds": 25.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 91.266, "latency": 26.956, "stderr": 0.768, "cost_per_test": 0.002248, "avg_input_tokens": 1643.4, "avg_output_tokens": 3510.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 27.0, "median_query_seconds": 11.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-3": { "accuracy": 91.192, "latency": 14.214, "stderr": 0.771, "cost_per_test": 0.022041, "avg_input_tokens": 1642.4, "avg_output_tokens": 1141.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 14.2, "median_query_seconds": 10.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 91.118, "latency": 9.519, "stderr": 0.774, "cost_per_test": 0.000398, "avg_input_tokens": 1793.4, "avg_output_tokens": 2601.2, "avg_reasoning_tokens": 2053.9, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 9.5, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 91.044, "latency": 3.386, "stderr": 0.777, "cost_per_test": 0.000703, "avg_input_tokens": 1793.4, "avg_output_tokens": 1309.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 3.4, "median_query_seconds": 2.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 91.044, "latency": 23.543, "stderr": 0.8, "cost_per_test": 0.001944, "avg_input_tokens": 1663.6, "avg_output_tokens": 496.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 23.5, "median_query_seconds": 18.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 90.97, "latency": 48.241, "stderr": 0.78, "cost_per_test": 0.037224, "avg_input_tokens": 1756.9, "avg_output_tokens": 4611.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1333, "avg_turns": 1.0, "avg_query_seconds": 48.2, "median_query_seconds": 39.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 90.896, "latency": 9.118, "stderr": 0.783, "cost_per_test": 0.013312, "avg_input_tokens": 1951.2, "avg_output_tokens": 497.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 9.1, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 90.896, "latency": 9.175, "stderr": 0.886, "cost_per_test": 0.000463, "avg_input_tokens": 1706.4, "avg_output_tokens": 945.3, "avg_reasoning_tokens": 730.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 9.2, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-0709": { "accuracy": 90.822, "latency": 57.184, "stderr": 0.786, "cost_per_test": 0.051539, "avg_input_tokens": 1643.4, "avg_output_tokens": 3107.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 57.2, "median_query_seconds": 22.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 90.674, "latency": 4.803, "stderr": 0.791, "cost_per_test": 0.001774, "avg_input_tokens": 1706.6, "avg_output_tokens": 682.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 4.8, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 90.6, "latency": 7.892, "stderr": 0.794, "cost_per_test": 0.001214, "avg_input_tokens": 1643.4, "avg_output_tokens": 1442.7, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 7.9, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 90.6, "latency": 29.009, "stderr": 0.794, "cost_per_test": 0.001272, "avg_input_tokens": 1792.4, "avg_output_tokens": 2296.4, "avg_reasoning_tokens": 2107.7, "avg_cache_read_tokens": 1567.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 29.0, "median_query_seconds": 11.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 89.637, "latency": 8.747, "stderr": 0.829, "cost_per_test": 0.010057, "avg_input_tokens": 1706.6, "avg_output_tokens": 830.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 8.7, "median_query_seconds": 3.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 89.563, "latency": 5.065, "stderr": 0.832, "cost_per_test": 0.000803, "avg_input_tokens": 1656.1, "avg_output_tokens": 498.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 5.1, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 88.601, "latency": 7.219, "stderr": 0.865, "cost_per_test": 0.002105, "avg_input_tokens": 1753.4, "avg_output_tokens": 3508.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 7.2, "median_query_seconds": 3.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 88.379, "latency": 15.539, "stderr": 1.232, "cost_per_test": 0.001912, "avg_input_tokens": 1744.9, "avg_output_tokens": 693.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 15.5, "median_query_seconds": 9.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 87.935, "latency": 20.684, "stderr": 1.014, "cost_per_test": 0.000939, "avg_input_tokens": 1773.6, "avg_output_tokens": 2715.4, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 20.7, "median_query_seconds": 16.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 87.935, "latency": 27.447, "stderr": 0.886, "cost_per_test": 0.003777, "avg_input_tokens": 1675.2, "avg_output_tokens": 700.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 27.4, "median_query_seconds": 16.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-4.5": { "accuracy": 87.861, "latency": 115.062, "stderr": 0.889, "cost_per_test": 0.012745, "avg_input_tokens": 1706.3, "avg_output_tokens": 5328.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 115.1, "median_query_seconds": 76.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 86.158, "latency": 6.24, "stderr": 0.94, "cost_per_test": 0.012018, "avg_input_tokens": 1951.2, "avg_output_tokens": 411.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 6.2, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 85.936, "latency": 4.791, "stderr": 0.946, "cost_per_test": 0.003813, "avg_input_tokens": 1793.4, "avg_output_tokens": 314.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 4.8, "median_query_seconds": 3.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 84.826, "latency": 16.389, "stderr": 0.976, "cost_per_test": 0.001766, "avg_input_tokens": 1663.6, "avg_output_tokens": 299.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 16.4, "median_query_seconds": 9.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 84.752, "latency": 23.459, "stderr": 1.217, "cost_per_test": 0.020505, "avg_input_tokens": 1979.2, "avg_output_tokens": 3705.1, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 23.5, "median_query_seconds": 9.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 83.79, "latency": 22.856, "stderr": 1.003, "cost_per_test": 0.011456, "avg_input_tokens": 1793.4, "avg_output_tokens": 921.5, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 22.9, "median_query_seconds": 14.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 82.902, "latency": 14.004, "stderr": 1.026, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 81.051, "latency": 3.881, "stderr": 1.066, "cost_per_test": 0.000279, "avg_input_tokens": 1804.4, "avg_output_tokens": 299.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1539.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 3.9, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 81.051, "latency": 6.366, "stderr": 1.066, "cost_per_test": 0.010386, "avg_input_tokens": 1951.2, "avg_output_tokens": 302.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 6.4, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-2-1212": { "accuracy": 80.977, "latency": 12.63, "stderr": 1.068, "cost_per_test": 0.008117, "avg_input_tokens": 1715.9, "avg_output_tokens": 468.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 12.6, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 80.755, "latency": 8.408, "stderr": 1.073, "cost_per_test": 0.000825, "avg_input_tokens": 1793.4, "avg_output_tokens": 1613.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 8.4, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 79.201, "latency": 2.126, "stderr": 1.113, "cost_per_test": 0.000223, "avg_input_tokens": 1793.4, "avg_output_tokens": 296.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 2.1, "median_query_seconds": 1.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 78.905, "latency": 11.455, "stderr": 1.11, "cost_per_test": 0.009151, "avg_input_tokens": 1706.6, "avg_output_tokens": 488.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 11.5, "median_query_seconds": 8.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 78.46, "latency": 13.483, "stderr": 1.133, "cost_per_test": 0.009716, "avg_input_tokens": 1839.0, "avg_output_tokens": 511.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 13.5, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 78.238, "latency": 2.936, "stderr": 1.162, "cost_per_test": 0.000442, "avg_input_tokens": 1706.6, "avg_output_tokens": 677.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 2.9, "median_query_seconds": 2.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 75.944, "latency": 5.799, "stderr": 1.163, "cost_per_test": 0.0006, "avg_input_tokens": 1656.1, "avg_output_tokens": 511.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 5.8, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 75.87, "latency": 15.477, "stderr": 1.189, "cost_per_test": null, "avg_input_tokens": 1727.7, "avg_output_tokens": 623.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 15.5, "median_query_seconds": 13.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 75.722, "latency": 10.179, "stderr": 1.167, "cost_per_test": 0.006698, "avg_input_tokens": 1865.2, "avg_output_tokens": 494.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 10.2, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 75.5, "latency": 3.932, "stderr": 1.17, "cost_per_test": 0.001958, "avg_input_tokens": 1734.7, "avg_output_tokens": 489.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 3.9, "median_query_seconds": 2.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4.3": { "accuracy": 74.538, "latency": 332.566, "stderr": 0.739, "cost_per_test": 0.008528, "avg_input_tokens": 1648.6, "avg_output_tokens": 3142.0, "avg_reasoning_tokens": 2880.4, "avg_cache_read_tokens": 1321.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1055, "avg_turns": 1.0, "avg_query_seconds": 332.6, "median_query_seconds": 311.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 74.241, "latency": 6.517, "stderr": 1.19, "cost_per_test": 0.000567, "avg_input_tokens": 1706.6, "avg_output_tokens": 519.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 6.5, "median_query_seconds": 5.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 73.871, "latency": 10.95, "stderr": 1.263, "cost_per_test": 0.009548, "avg_input_tokens": 1706.6, "avg_output_tokens": 528.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 11.0, "median_query_seconds": 8.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 73.205, "latency": 4.976, "stderr": 1.205, "cost_per_test": 0.000274, "avg_input_tokens": 1744.8, "avg_output_tokens": 477.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 5.0, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 72.835, "latency": 6.125, "stderr": 1.21, "cost_per_test": 0.000645, "avg_input_tokens": 1744.8, "avg_output_tokens": 493.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 6.1, "median_query_seconds": 4.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 68.246, "latency": 45.92, "stderr": 1.306, "cost_per_test": 0.000619, "avg_input_tokens": 1785.9, "avg_output_tokens": 2607.9, "avg_reasoning_tokens": 2282.1, "avg_cache_read_tokens": 1617.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 45.9, "median_query_seconds": 22.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 67.876, "latency": 1.575, "stderr": 1.27, "cost_per_test": 0.001346, "avg_input_tokens": 1763.4, "avg_output_tokens": 1986.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 1.6, "median_query_seconds": 1.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 66.173, "latency": 5.628, "stderr": 1.287, "cost_per_test": 0.002685, "avg_input_tokens": 1951.2, "avg_output_tokens": 281.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 5.6, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "poolside/laguna-m.1": { "accuracy": 59.511, "latency": 119.023, "stderr": 1.339, "cost_per_test": 0.002116, "avg_input_tokens": 1748.9, "avg_output_tokens": 9318.8, "avg_reasoning_tokens": 4505.3, "avg_cache_read_tokens": 1590.5, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 119.0, "median_query_seconds": 69.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 58.105, "latency": 16.561, "stderr": 1.345, "cost_per_test": 0.006397, "avg_input_tokens": 1849.3, "avg_output_tokens": 337.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 16.6, "median_query_seconds": 11.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 56.477, "latency": 44.146, "stderr": 1.349, "cost_per_test": null, "avg_input_tokens": 1727.7, "avg_output_tokens": 2186.6, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 44.1, "median_query_seconds": 30.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 49.741, "latency": 33.17, "stderr": 1.36, "cost_per_test": 0.015989, "avg_input_tokens": 1744.8, "avg_output_tokens": 2499.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 33.2, "median_query_seconds": 17.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 39.97, "latency": 33.945, "stderr": 1.333, "cost_per_test": 0.00496, "avg_input_tokens": 1744.9, "avg_output_tokens": 2735.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 33.9, "median_query_seconds": 26.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-r-plus": { "accuracy": 39.008, "latency": 4.218, "stderr": 1.327, "cost_per_test": 0.006778, "avg_input_tokens": 1839.0, "avg_output_tokens": 218.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 4.2, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 30.792, "latency": 4.284, "stderr": 1.256, "cost_per_test": 0.000583, "avg_input_tokens": 1849.3, "avg_output_tokens": 533.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1351, "avg_turns": 1.0, "avg_query_seconds": 4.3, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "other": { "anthropic/claude-fable-5-1": { "accuracy": 89.935, "latency": 22.847, "stderr": 0.99, "cost_per_test": 0.096818, "avg_input_tokens": 1374.0, "avg_output_tokens": 1593.1, "avg_cache_write_tokens": 1370.0, "turn_metrics": { "tasks_measured": 922, "avg_turns": 1.0, "avg_query_seconds": 22.8, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 89.827, "latency": 14.291, "stderr": 0.994, "cost_per_test": 0.052645, "avg_input_tokens": 1372.0, "avg_output_tokens": 778.6, "avg_cache_write_tokens": 43.6, "turn_metrics": { "tasks_measured": 922, "avg_turns": 1.0, "avg_query_seconds": 14.3, "median_query_seconds": 9.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 88.636, "latency": 15.066, "stderr": 1.044, "cost_per_test": 0.016267, "avg_input_tokens": 935.1, "avg_output_tokens": 1199.8, "avg_reasoning_tokens": 1042.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 15.1, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 87.987, "latency": 7.209, "stderr": 1.07, "cost_per_test": 0.019263, "avg_input_tokens": 1372.0, "avg_output_tokens": 427.6, "avg_reasoning_tokens": 268.3, "avg_cache_write_tokens": 1370.0, "turn_metrics": { "tasks_measured": 922, "avg_turns": 1.0, "avg_query_seconds": 7.2, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 87.987, "latency": 16.616, "stderr": 1.07, "cost_per_test": 0.024161, "avg_input_tokens": 940.0, "avg_output_tokens": 1856.7, "avg_reasoning_tokens": 1632.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 16.6, "median_query_seconds": 10.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 87.771, "latency": 14.998, "stderr": 1.078, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 87.771, "latency": 16.333, "stderr": 1.078, "cost_per_test": 0.068106, "avg_input_tokens": 1026.6, "avg_output_tokens": 702.8, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 16.3, "median_query_seconds": 15.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 87.662, "latency": 10.365, "stderr": 1.082, "cost_per_test": 0.023076, "avg_input_tokens": 897.6, "avg_output_tokens": 619.6, "avg_reasoning_tokens": 573.9, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 10.4, "median_query_seconds": 4.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 87.554, "latency": 3.375, "stderr": 1.086, "cost_per_test": 0.00772, "avg_input_tokens": 935.1, "avg_output_tokens": 842.2, "avg_reasoning_tokens": 723.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 3.4, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 87.554, "latency": 7.512, "stderr": 1.086, "cost_per_test": 0.019777, "avg_input_tokens": 935.1, "avg_output_tokens": 2449.9, "avg_reasoning_tokens": 2307.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 7.5, "median_query_seconds": 2.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 87.554, "latency": 37.383, "stderr": 1.118, "cost_per_test": 0.016173, "avg_input_tokens": 935.1, "avg_output_tokens": 1191.9, "avg_reasoning_tokens": 1030.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 37.4, "median_query_seconds": 16.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 87.338, "latency": 8.322, "stderr": 1.094, "cost_per_test": 0.033476, "avg_input_tokens": 998.6, "avg_output_tokens": 246.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 8.3, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 87.229, "latency": 42.931, "stderr": 1.098, "cost_per_test": 0.012286, "avg_input_tokens": 924.7, "avg_output_tokens": 1943.5, "avg_reasoning_tokens": 1809.8, "avg_cache_read_tokens": 816.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 42.9, "median_query_seconds": 16.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.5": { "accuracy": 87.229, "latency": 310.979, "stderr": 1.11, "cost_per_test": 0.010007, "avg_input_tokens": 924.7, "avg_output_tokens": 1549.7, "avg_reasoning_tokens": 1381.7, "avg_cache_read_tokens": 760.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 311.0, "median_query_seconds": 226.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 86.905, "latency": 17.063, "stderr": 1.635, "cost_per_test": 0.016102, "avg_input_tokens": 1015.6, "avg_output_tokens": 870.4, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 17.1, "median_query_seconds": 12.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 86.905, "latency": 24.514, "stderr": 1.11, "cost_per_test": 0.012398, "avg_input_tokens": 935.1, "avg_output_tokens": 3976.7, "avg_reasoning_tokens": 3780.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 24.5, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 86.905, "latency": 35.837, "stderr": 1.162, "cost_per_test": 0.031505, "avg_input_tokens": 1376.4, "avg_output_tokens": 984.8, "avg_cache_write_tokens": 2.3, "turn_metrics": { "tasks_measured": 919, "avg_turns": 1.0, "avg_query_seconds": 35.8, "median_query_seconds": 26.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 86.797, "latency": 33.861, "stderr": 1.114, "cost_per_test": 0.023843, "avg_input_tokens": 897.6, "avg_output_tokens": 1175.0, "avg_reasoning_tokens": 1108.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 33.9, "median_query_seconds": 19.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 86.472, "latency": 5.755, "stderr": 1.125, "cost_per_test": 0.009441, "avg_input_tokens": 935.1, "avg_output_tokens": 1071.7, "avg_reasoning_tokens": 920.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 5.8, "median_query_seconds": 4.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 86.472, "latency": 26.512, "stderr": 1.564, "cost_per_test": 0.051891, "avg_input_tokens": 897.6, "avg_output_tokens": 1697.1, "avg_reasoning_tokens": 1612.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 26.5, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 86.147, "latency": 13.617, "stderr": 1.136, "cost_per_test": 0.030976, "avg_input_tokens": 1371.8, "avg_output_tokens": 968.6, "avg_cache_write_tokens": 2.3, "turn_metrics": { "tasks_measured": 921, "avg_turns": 1.0, "avg_query_seconds": 13.6, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 86.039, "latency": 19.902, "stderr": 1.14, "cost_per_test": 0.010569, "avg_input_tokens": 957.1, "avg_output_tokens": 1090.2, "avg_reasoning_tokens": 901.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 19.9, "median_query_seconds": 17.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 86.039, "latency": 26.158, "stderr": 1.14, "cost_per_test": 0.001852, "avg_input_tokens": 916.2, "avg_output_tokens": 1268.9, "avg_cache_read_tokens": 39.7, "avg_cache_write_tokens": 874.6, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 26.2, "median_query_seconds": 14.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 85.823, "latency": 22.934, "stderr": 1.22, "cost_per_test": 0.020915, "avg_input_tokens": 1026.6, "avg_output_tokens": 1189.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 22.9, "median_query_seconds": 16.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 85.714, "latency": 34.123, "stderr": 1.151, "cost_per_test": 0.00613, "avg_input_tokens": 957.2, "avg_output_tokens": 1883.7, "avg_reasoning_tokens": 1736.9, "turn_metrics": { "tasks_measured": 921, "avg_turns": 1.0, "avg_query_seconds": 34.1, "median_query_seconds": 33.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 85.498, "latency": 19.814, "stderr": 1.176, "cost_per_test": 0.099581, "avg_input_tokens": 1027.6, "avg_output_tokens": 1122.2, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 19.8, "median_query_seconds": 14.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 85.39, "latency": 26.94, "stderr": 1.22, "cost_per_test": 0.043786, "avg_input_tokens": 1371.8, "avg_output_tokens": 2576.2, "avg_cache_write_tokens": 1369.8, "turn_metrics": { "tasks_measured": 921, "avg_turns": 1.0, "avg_query_seconds": 26.9, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 85.281, "latency": 16.9, "stderr": 1.18, "cost_per_test": 0.008698, "avg_input_tokens": 1056.4, "avg_output_tokens": 1956.9, "avg_reasoning_tokens": 1819.7, "avg_cache_read_tokens": 853.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 16.9, "median_query_seconds": 12.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 85.281, "latency": 45.914, "stderr": 1.166, "cost_per_test": 0.014219, "avg_input_tokens": 995.1, "avg_output_tokens": 2038.2, "avg_reasoning_tokens": 1948.7, "turn_metrics": { "tasks_measured": 923, "avg_turns": 1.0, "avg_query_seconds": 45.9, "median_query_seconds": 11.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 84.957, "latency": 12.591, "stderr": 1.18, "cost_per_test": 0.010452, "avg_input_tokens": 897.6, "avg_output_tokens": 933.0, "avg_reasoning_tokens": 830.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 12.6, "median_query_seconds": 8.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 84.957, "latency": 22.307, "stderr": 1.176, "cost_per_test": 0.015416, "avg_input_tokens": 897.6, "avg_output_tokens": 1429.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 22.3, "median_query_seconds": 16.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 84.957, "latency": 35.056, "stderr": 1.183, "cost_per_test": 0.00837, "avg_input_tokens": 899.2, "avg_output_tokens": 1888.6, "avg_reasoning_tokens": 1731.6, "avg_cache_read_tokens": 710.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 922, "avg_turns": 1.0, "avg_query_seconds": 35.1, "median_query_seconds": 19.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 84.957, "latency": 98.047, "stderr": 1.176, "cost_per_test": 0.011346, "avg_input_tokens": 906.1, "avg_output_tokens": 2779.3, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 753.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 98.0, "median_query_seconds": 48.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 84.632, "latency": 6.863, "stderr": 1.186, "cost_per_test": 0.035681, "avg_input_tokens": 998.6, "avg_output_tokens": 276.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 6.9, "median_query_seconds": 7.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "meta/muse_spark": { "accuracy": 84.632, "latency": 32.647, "stderr": 1.186, "cost_per_test": 0.001278, "avg_input_tokens": 954.4, "avg_output_tokens": 1891.6, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 32.6, "median_query_seconds": 28.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.1": { "accuracy": 84.524, "latency": 36.629, "stderr": 1.19, "cost_per_test": 0.005805, "avg_input_tokens": 913.2, "avg_output_tokens": 1717.0, "avg_reasoning_tokens": 1525.9, "avg_cache_read_tokens": 753.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 36.6, "median_query_seconds": 17.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 84.524, "latency": 45.323, "stderr": 1.203, "cost_per_test": 0.008369, "avg_input_tokens": 992.3, "avg_output_tokens": 2029.7, "avg_reasoning_tokens": 1971.1, "avg_cache_read_tokens": 766.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 45.3, "median_query_seconds": 11.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 84.524, "latency": 54.257, "stderr": 1.19, "cost_per_test": 0.014785, "avg_input_tokens": 920.2, "avg_output_tokens": 3189.9, "avg_reasoning_tokens": 3064.3, "avg_cache_read_tokens": 472.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 54.3, "median_query_seconds": 8.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-0709": { "accuracy": 84.524, "latency": 83.034, "stderr": 1.19, "cost_per_test": 0.031888, "avg_input_tokens": 885.7, "avg_output_tokens": 1948.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 83.0, "median_query_seconds": 45.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 84.416, "latency": 9.539, "stderr": 1.193, "cost_per_test": 0.007168, "avg_input_tokens": 897.6, "avg_output_tokens": 671.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 9.5, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 84.307, "latency": 7.093, "stderr": 1.197, "cost_per_test": 0.008299, "avg_input_tokens": 1006.3, "avg_output_tokens": 1332.0, "avg_reasoning_tokens": 1231.1, "avg_cache_read_tokens": 947.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 7.1, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 84.199, "latency": 53.559, "stderr": 1.2, "cost_per_test": 0.006713, "avg_input_tokens": 913.3, "avg_output_tokens": 1997.9, "avg_cache_read_tokens": 732.2, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 53.6, "median_query_seconds": 36.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 84.199, "latency": 54.199, "stderr": 1.2, "cost_per_test": 0.027172, "avg_input_tokens": 905.6, "avg_output_tokens": 2790.9, "avg_cache_read_tokens": 781.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 54.2, "median_query_seconds": 12.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.3": { "accuracy": 84.199, "latency": 336.051, "stderr": 1.2, "cost_per_test": 0.00688, "avg_input_tokens": 891.7, "avg_output_tokens": 2619.1, "avg_reasoning_tokens": 2414.3, "avg_cache_read_tokens": 744.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 336.1, "median_query_seconds": 311.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 84.199, "latency": 420.745, "stderr": 1.2, "cost_per_test": 0.013978, "avg_input_tokens": 979.3, "avg_output_tokens": 3371.3, "avg_reasoning_tokens": 3265.2, "avg_cache_read_tokens": 521.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 420.7, "median_query_seconds": 497.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 84.091, "latency": 5.556, "stderr": 1.203, "cost_per_test": 0.006067, "avg_input_tokens": 897.6, "avg_output_tokens": 356.0, "avg_reasoning_tokens": 312.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 5.6, "median_query_seconds": 3.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 83.983, "latency": 30.907, "stderr": 1.207, "cost_per_test": 0.012905, "avg_input_tokens": 984.2, "avg_output_tokens": 823.0, "avg_reasoning_tokens": 676.9, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 30.9, "median_query_seconds": 16.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.2": { "accuracy": 83.658, "latency": 34.143, "stderr": 1.216, "cost_per_test": 0.008226, "avg_input_tokens": 920.2, "avg_output_tokens": 1717.6, "avg_reasoning_tokens": 1574.5, "avg_cache_read_tokens": 535.9, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 34.1, "median_query_seconds": 16.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 83.55, "latency": 21.986, "stderr": 1.22, "cost_per_test": 0.021591, "avg_input_tokens": 1026.6, "avg_output_tokens": 1234.1, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 22.0, "median_query_seconds": 15.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 83.333, "latency": 24.362, "stderr": 1.226, "cost_per_test": 0.004649, "avg_input_tokens": 906.2, "avg_output_tokens": 1493.9, "avg_cache_read_tokens": 753.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 24.4, "median_query_seconds": 17.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 83.225, "latency": 12.88, "stderr": 1.229, "cost_per_test": 0.001856, "avg_input_tokens": 979.3, "avg_output_tokens": 1313.4, "avg_reasoning_tokens": 1266.6, "avg_cache_read_tokens": 723.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 12.9, "median_query_seconds": 4.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 83.225, "latency": 19.935, "stderr": 1.229, "cost_per_test": 0.022639, "avg_input_tokens": 1026.6, "avg_output_tokens": 1303.9, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 19.9, "median_query_seconds": 14.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 83.117, "latency": 3.379, "stderr": 1.232, "cost_per_test": 0.002828, "avg_input_tokens": 935.1, "avg_output_tokens": 1018.8, "avg_reasoning_tokens": 891.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 3.4, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 83.117, "latency": 21.251, "stderr": 1.401, "cost_per_test": 0.022672, "avg_input_tokens": 1015.6, "avg_output_tokens": 1308.4, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 21.3, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 82.9, "latency": 15.648, "stderr": 1.239, "cost_per_test": null, "avg_input_tokens": 952.9, "avg_output_tokens": 1410.1, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 15.6, "median_query_seconds": 7.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 82.684, "latency": 14.438, "stderr": 1.278, "cost_per_test": 0.002099, "avg_input_tokens": 897.6, "avg_output_tokens": 1599.9, "avg_reasoning_tokens": 1549.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 14.4, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 82.684, "latency": 16.101, "stderr": 1.245, "cost_per_test": 0.081239, "avg_input_tokens": 897.6, "avg_output_tokens": 1129.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 16.1, "median_query_seconds": 12.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 82.576, "latency": 12.065, "stderr": 1.248, "cost_per_test": 0.005927, "avg_input_tokens": 934.0, "avg_output_tokens": 476.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 12.1, "median_query_seconds": 11.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 82.468, "latency": 23.566, "stderr": 1.251, "cost_per_test": 0.000529, "avg_input_tokens": 920.2, "avg_output_tokens": 1985.7, "avg_cache_read_tokens": 610.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 23.6, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 82.468, "latency": 59.92, "stderr": 1.251, "cost_per_test": 0.000662, "avg_input_tokens": 900.1, "avg_output_tokens": 1469.5, "avg_reasoning_tokens": 1394.5, "avg_cache_read_tokens": 810.4, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 59.9, "median_query_seconds": 44.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 82.468, "latency": 62.772, "stderr": 1.251, "cost_per_test": 0.0061, "avg_input_tokens": 957.1, "avg_output_tokens": 2382.2, "avg_reasoning_tokens": 2166.7, "turn_metrics": { "tasks_measured": 920, "avg_turns": 1.0, "avg_query_seconds": 62.8, "median_query_seconds": 47.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 82.359, "latency": 23.394, "stderr": 1.278, "cost_per_test": 0.024895, "avg_input_tokens": 897.6, "avg_output_tokens": 1666.0, "avg_reasoning_tokens": 1580.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 23.4, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 82.035, "latency": 80.23, "stderr": 1.263, "cost_per_test": 0.017908, "avg_input_tokens": 942.0, "avg_output_tokens": 2796.3, "avg_reasoning_tokens": 2319.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 923, "avg_turns": 1.0, "avg_query_seconds": 80.2, "median_query_seconds": 62.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 81.926, "latency": 5.446, "stderr": 1.266, "cost_per_test": 0.000674, "avg_input_tokens": 935.1, "avg_output_tokens": 1452.2, "avg_reasoning_tokens": 1274.2, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 5.4, "median_query_seconds": 3.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 81.71, "latency": 6.537, "stderr": 1.272, "cost_per_test": 0.007223, "avg_input_tokens": 998.6, "avg_output_tokens": 281.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 6.5, "median_query_seconds": 6.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 81.602, "latency": 38.008, "stderr": 1.357, "cost_per_test": 0.000157, "avg_input_tokens": 900.3, "avg_output_tokens": 259.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 808.7, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 38.0, "median_query_seconds": 28.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 81.385, "latency": 18.175, "stderr": 1.494, "cost_per_test": 0.008235, "avg_input_tokens": 899.3, "avg_output_tokens": 1300.2, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 18.2, "median_query_seconds": 12.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 81.277, "latency": 18.963, "stderr": 1.289, "cost_per_test": 0.002008, "avg_input_tokens": 1054.2, "avg_output_tokens": 749.3, "avg_cache_read_tokens": 880.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 19.0, "median_query_seconds": 12.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 81.277, "latency": 45.035, "stderr": 1.283, "cost_per_test": 0.003006, "avg_input_tokens": 905.6, "avg_output_tokens": 2437.2, "avg_cache_read_tokens": 793.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 923, "avg_turns": 1.0, "avg_query_seconds": 45.0, "median_query_seconds": 13.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 81.169, "latency": 5.192, "stderr": 1.286, "cost_per_test": 0.006876, "avg_input_tokens": 998.6, "avg_output_tokens": 258.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 5.2, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 81.061, "latency": 15.741, "stderr": 1.289, "cost_per_test": 0.000682, "avg_input_tokens": 1034.7, "avg_output_tokens": 1206.7, "avg_reasoning_tokens": 1128.2, "avg_cache_read_tokens": 854.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 15.7, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 80.844, "latency": 28.347, "stderr": 1.295, "cost_per_test": 0.003492, "avg_input_tokens": 939.9, "avg_output_tokens": 393.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 28.3, "median_query_seconds": 24.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 80.628, "latency": 36.392, "stderr": 1.582, "cost_per_test": 0.000878, "avg_input_tokens": 1182.6, "avg_output_tokens": 917.4, "avg_reasoning_tokens": 767.8, "avg_cache_read_tokens": 1002.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 36.4, "median_query_seconds": 20.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3-max": { "accuracy": 80.519, "latency": 0.0, "stderr": 1.303, "cost_per_test": 0.003535, "avg_input_tokens": 939.9, "avg_output_tokens": 401.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 80.411, "latency": 16.063, "stderr": 1.322, "cost_per_test": 0.002935, "avg_input_tokens": 897.6, "avg_output_tokens": 1355.4, "avg_reasoning_tokens": 1285.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 16.1, "median_query_seconds": 6.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 80.411, "latency": 67.872, "stderr": 1.33, "cost_per_test": 0.001378, "avg_input_tokens": 905.9, "avg_output_tokens": 1861.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 67.9, "median_query_seconds": 36.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-4.5": { "accuracy": 80.411, "latency": 69.032, "stderr": 1.306, "cost_per_test": 0.005996, "avg_input_tokens": 913.2, "avg_output_tokens": 2476.4, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 69.0, "median_query_seconds": 59.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 80.303, "latency": 4.209, "stderr": 1.308, "cost_per_test": 0.000805, "avg_input_tokens": 934.0, "avg_output_tokens": 915.2, "avg_reasoning_tokens": 705.4, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 4.2, "median_query_seconds": 3.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-4.7": { "accuracy": 80.303, "latency": 72.709, "stderr": 1.308, "cost_per_test": 0.004031, "avg_input_tokens": 913.2, "avg_output_tokens": 1708.7, "avg_cache_read_tokens": 562.7, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 72.7, "median_query_seconds": 35.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 79.978, "latency": 18.197, "stderr": 1.324, "cost_per_test": 0.000405, "avg_input_tokens": 1178.9, "avg_output_tokens": 1356.9, "avg_reasoning_tokens": 1250.0, "avg_cache_read_tokens": 1018.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 18.2, "median_query_seconds": 11.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 79.87, "latency": 2.19, "stderr": 1.319, "cost_per_test": 0.00323, "avg_input_tokens": 898.6, "avg_output_tokens": 179.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 2.2, "median_query_seconds": 1.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 79.654, "latency": 4.164, "stderr": 1.324, "cost_per_test": 0.001093, "avg_input_tokens": 995.7, "avg_output_tokens": 1787.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 4.2, "median_query_seconds": 2.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 79.545, "latency": 5.76, "stderr": 1.327, "cost_per_test": 0.006445, "avg_input_tokens": 998.6, "avg_output_tokens": 229.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 5.8, "median_query_seconds": 5.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 79.545, "latency": 7.515, "stderr": 1.327, "cost_per_test": 0.000798, "avg_input_tokens": 934.0, "avg_output_tokens": 926.8, "avg_reasoning_tokens": 719.6, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 7.5, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 79.545, "latency": 7.928, "stderr": 1.327, "cost_per_test": 0.000975, "avg_input_tokens": 885.7, "avg_output_tokens": 1418.2, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 7.9, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 79.545, "latency": 14.719, "stderr": 1.327, "cost_per_test": 0.003481, "avg_input_tokens": 897.7, "avg_output_tokens": 1628.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 14.7, "median_query_seconds": 12.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.6": { "accuracy": 79.545, "latency": 34.003, "stderr": 1.327, "cost_per_test": 0.004223, "avg_input_tokens": 915.0, "avg_output_tokens": 1671.9, "turn_metrics": { "tasks_measured": 922, "avg_turns": 1.0, "avg_query_seconds": 34.0, "median_query_seconds": 22.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-3": { "accuracy": 79.221, "latency": 8.568, "stderr": 1.335, "cost_per_test": 0.011647, "avg_input_tokens": 884.7, "avg_output_tokens": 599.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 8.6, "median_query_seconds": 8.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 79.221, "latency": 8.956, "stderr": 1.632, "cost_per_test": 0.000929, "avg_input_tokens": 939.9, "avg_output_tokens": 306.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 9.0, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 78.788, "latency": 25.541, "stderr": 1.345, "cost_per_test": 0.001274, "avg_input_tokens": 957.6, "avg_output_tokens": 2945.0, "avg_reasoning_tokens": 2679.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 918, "avg_turns": 1.0, "avg_query_seconds": 25.5, "median_query_seconds": 21.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 78.788, "latency": 30.649, "stderr": 1.345, "cost_per_test": 0.003394, "avg_input_tokens": 919.2, "avg_output_tokens": 1275.6, "avg_cache_read_tokens": 36.0, "avg_cache_write_tokens": 883.2, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 30.6, "median_query_seconds": 20.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 78.68, "latency": 4.975, "stderr": 1.347, "cost_per_test": 0.000712, "avg_input_tokens": 885.7, "avg_output_tokens": 893.2, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 5.0, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 78.68, "latency": 7.297, "stderr": 1.347, "cost_per_test": 0.004122, "avg_input_tokens": 897.6, "avg_output_tokens": 712.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 7.3, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 78.247, "latency": 9.885, "stderr": 1.357, "cost_per_test": 0.000955, "avg_input_tokens": 899.3, "avg_output_tokens": 162.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 9.9, "median_query_seconds": 8.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 78.139, "latency": 39.948, "stderr": 1.36, "cost_per_test": 0.026514, "avg_input_tokens": 951.9, "avg_output_tokens": 3344.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 39.9, "median_query_seconds": 40.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 78.03, "latency": 40.542, "stderr": 1.364, "cost_per_test": 0.001379, "avg_input_tokens": 939.9, "avg_output_tokens": 1331.6, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 40.5, "median_query_seconds": 23.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 77.922, "latency": 10.042, "stderr": 1.364, "cost_per_test": 0.001608, "avg_input_tokens": 915.2, "avg_output_tokens": 231.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 10.0, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 77.706, "latency": 5.163, "stderr": 1.369, "cost_per_test": 0.000541, "avg_input_tokens": 902.4, "avg_output_tokens": 389.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 5.2, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 77.489, "latency": 36.909, "stderr": 1.386, "cost_per_test": 0.002513, "avg_input_tokens": 899.2, "avg_output_tokens": 1817.8, "avg_cache_read_tokens": 16.0, "avg_cache_write_tokens": 883.2, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 36.9, "median_query_seconds": 16.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 77.273, "latency": 5.526, "stderr": 1.397, "cost_per_test": 0.004344, "avg_input_tokens": 898.6, "avg_output_tokens": 209.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 5.5, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 77.165, "latency": 5.932, "stderr": 1.381, "cost_per_test": 0.004937, "avg_input_tokens": 898.6, "avg_output_tokens": 269.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 5.9, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 76.84, "latency": 8.282, "stderr": 1.388, "cost_per_test": 0.00019, "avg_input_tokens": 934.0, "avg_output_tokens": 1211.2, "avg_reasoning_tokens": 970.1, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 8.3, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 76.84, "latency": 20.061, "stderr": 1.388, "cost_per_test": 0.007396, "avg_input_tokens": 999.1, "avg_output_tokens": 2298.9, "avg_reasoning_tokens": 2212.2, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 20.1, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-2-1212": { "accuracy": 76.623, "latency": 3.619, "stderr": 1.392, "cost_per_test": 0.003518, "avg_input_tokens": 926.5, "avg_output_tokens": 166.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 3.6, "median_query_seconds": 2.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 76.407, "latency": 2.052, "stderr": 1.397, "cost_per_test": 0.0003, "avg_input_tokens": 934.0, "avg_output_tokens": 515.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 2.1, "median_query_seconds": 1.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 76.19, "latency": 9.149, "stderr": 1.401, "cost_per_test": 0.000353, "avg_input_tokens": 962.8, "avg_output_tokens": 1474.6, "avg_cache_read_tokens": 724.2, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 9.1, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 76.082, "latency": 3.419, "stderr": 1.41, "cost_per_test": 0.00077, "avg_input_tokens": 939.9, "avg_output_tokens": 197.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 3.4, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 76.082, "latency": 29.004, "stderr": 1.403, "cost_per_test": 0.009138, "avg_input_tokens": 939.9, "avg_output_tokens": 1457.7, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 29.0, "median_query_seconds": 19.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 75.758, "latency": 2.07, "stderr": 1.41, "cost_per_test": 0.000755, "avg_input_tokens": 898.6, "avg_output_tokens": 247.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 2.1, "median_query_seconds": 1.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 75.758, "latency": 14.09, "stderr": 1.64, "cost_per_test": 0.008857, "avg_input_tokens": 1026.6, "avg_output_tokens": 1566.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 14.1, "median_query_seconds": 9.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 75.325, "latency": 1.702, "stderr": 1.418, "cost_per_test": 0.000172, "avg_input_tokens": 934.0, "avg_output_tokens": 197.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 1.7, "median_query_seconds": 1.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 75.0, "latency": 1.857, "stderr": 1.425, "cost_per_test": 0.0016, "avg_input_tokens": 934.0, "avg_output_tokens": 86.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 1.9, "median_query_seconds": 1.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 74.351, "latency": 1.454, "stderr": 1.437, "cost_per_test": 0.000127, "avg_input_tokens": 1046.7, "avg_output_tokens": 88.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 839.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 1.5, "median_query_seconds": 1.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 74.351, "latency": 70.735, "stderr": 1.463, "cost_per_test": 0.000805, "avg_input_tokens": 983.3, "avg_output_tokens": 1730.4, "avg_reasoning_tokens": 1528.2, "avg_cache_read_tokens": 842.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 70.7, "median_query_seconds": 43.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 74.242, "latency": 17.041, "stderr": 1.439, "cost_per_test": 0.010356, "avg_input_tokens": 897.6, "avg_output_tokens": 2129.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 17.0, "median_query_seconds": 11.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 74.134, "latency": 4.106, "stderr": 1.441, "cost_per_test": 0.000891, "avg_input_tokens": 899.3, "avg_output_tokens": 90.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 4.1, "median_query_seconds": 3.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 73.81, "latency": 38.694, "stderr": 1.448, "cost_per_test": 0.001439, "avg_input_tokens": 965.6, "avg_output_tokens": 2157.4, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 38.7, "median_query_seconds": 30.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 73.701, "latency": 30.305, "stderr": 1.454, "cost_per_test": null, "avg_input_tokens": 937.1, "avg_output_tokens": 866.3, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 30.3, "median_query_seconds": 27.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 73.052, "latency": 4.498, "stderr": 1.46, "cost_per_test": 0.000764, "avg_input_tokens": 1005.7, "avg_output_tokens": 1125.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 4.5, "median_query_seconds": 1.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 72.619, "latency": 30.014, "stderr": 1.467, "cost_per_test": 0.00129, "avg_input_tokens": 897.6, "avg_output_tokens": 3111.8, "avg_reasoning_tokens": 3032.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 30.0, "median_query_seconds": 25.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 71.861, "latency": 5.742, "stderr": 1.557, "cost_per_test": 0.000192, "avg_input_tokens": 897.8, "avg_output_tokens": 367.8, "avg_reasoning_tokens": 287.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 71.429, "latency": 3.776, "stderr": 1.486, "cost_per_test": 0.001113, "avg_input_tokens": 941.1, "avg_output_tokens": 323.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 3.8, "median_query_seconds": 3.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 70.671, "latency": 3.589, "stderr": 1.498, "cost_per_test": 0.003046, "avg_input_tokens": 1037.8, "avg_output_tokens": 161.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 3.6, "median_query_seconds": 2.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 69.481, "latency": 6.18, "stderr": 1.563, "cost_per_test": 0.005133, "avg_input_tokens": 938.3, "avg_output_tokens": 278.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 6.2, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 67.424, "latency": 26.896, "stderr": 1.617, "cost_per_test": 0.000284, "avg_input_tokens": 1020.3, "avg_output_tokens": 1129.4, "avg_reasoning_tokens": 945.8, "avg_cache_read_tokens": 875.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 26.9, "median_query_seconds": 21.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 66.991, "latency": 5.582, "stderr": 1.547, "cost_per_test": 0.001749, "avg_input_tokens": 998.6, "avg_output_tokens": 237.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 5.6, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 66.775, "latency": 12.585, "stderr": 1.619, "cost_per_test": null, "avg_input_tokens": 934.1, "avg_output_tokens": 543.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 12.6, "median_query_seconds": 11.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 66.45, "latency": 1.757, "stderr": 1.553, "cost_per_test": 0.00012, "avg_input_tokens": 939.9, "avg_output_tokens": 163.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 1.8, "median_query_seconds": 1.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 66.45, "latency": 2.835, "stderr": 1.553, "cost_per_test": 0.000253, "avg_input_tokens": 898.6, "avg_output_tokens": 196.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 2.8, "median_query_seconds": 2.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 66.126, "latency": 3.433, "stderr": 1.557, "cost_per_test": 0.000353, "avg_input_tokens": 902.4, "avg_output_tokens": 322.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 3.4, "median_query_seconds": 3.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 66.126, "latency": 16.455, "stderr": 1.576, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 65.043, "latency": 2.356, "stderr": 1.569, "cost_per_test": 0.0003, "avg_input_tokens": 939.9, "avg_output_tokens": 186.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 2.4, "median_query_seconds": 1.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 64.394, "latency": 1.481, "stderr": 1.577, "cost_per_test": 0.000196, "avg_input_tokens": 898.6, "avg_output_tokens": 265.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 1.5, "median_query_seconds": 1.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 62.338, "latency": 1.045, "stderr": 1.594, "cost_per_test": 0.000108, "avg_input_tokens": 934.0, "avg_output_tokens": 127.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 1.0, "median_query_seconds": 0.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 54.545, "latency": 9.55, "stderr": 1.638, "cost_per_test": 0.002112, "avg_input_tokens": 939.9, "avg_output_tokens": 1094.4, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 9.6, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-r-plus": { "accuracy": 51.082, "latency": 1.729, "stderr": 1.645, "cost_per_test": 0.003163, "avg_input_tokens": 938.3, "avg_output_tokens": 81.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 1.7, "median_query_seconds": 1.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 44.048, "latency": 4.044, "stderr": 1.629, "cost_per_test": 0.002706, "avg_input_tokens": 978.8, "avg_output_tokens": 93.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 4.0, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 41.991, "latency": 1.232, "stderr": 1.624, "cost_per_test": 0.00023, "avg_input_tokens": 978.8, "avg_output_tokens": 84.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 924, "avg_turns": 1.0, "avg_query_seconds": 1.2, "median_query_seconds": 1.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "philosophy": { "anthropic/claude-fable-5": { "accuracy": 90.581, "latency": 24.035, "stderr": 1.308, "cost_per_test": 0.069071, "avg_input_tokens": 1399.9, "avg_output_tokens": 1101.0, "avg_cache_write_tokens": 8.5, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 24.0, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 90.381, "latency": 15.572, "stderr": 1.32, "cost_per_test": 0.017428, "avg_input_tokens": 974.2, "avg_output_tokens": 1290.0, "avg_reasoning_tokens": 1086.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 15.6, "median_query_seconds": 9.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 90.18, "latency": 27.85, "stderr": 1.332, "cost_per_test": 0.117235, "avg_input_tokens": 1402.6, "avg_output_tokens": 1994.3, "avg_cache_write_tokens": 1398.6, "turn_metrics": { "tasks_measured": 493, "avg_turns": 1.0, "avg_query_seconds": 27.9, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 89.379, "latency": 3.821, "stderr": 1.379, "cost_per_test": 0.00827, "avg_input_tokens": 974.2, "avg_output_tokens": 907.8, "avg_reasoning_tokens": 756.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 3.8, "median_query_seconds": 2.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 89.379, "latency": 24.329, "stderr": 1.391, "cost_per_test": 0.124834, "avg_input_tokens": 1097.7, "avg_output_tokens": 1444.9, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 24.3, "median_query_seconds": 16.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 89.178, "latency": 21.703, "stderr": 1.391, "cost_per_test": 0.01142, "avg_input_tokens": 984.9, "avg_output_tokens": 1194.3, "avg_reasoning_tokens": 962.8, "turn_metrics": { "tasks_measured": 496, "avg_turns": 1.0, "avg_query_seconds": 21.7, "median_query_seconds": 19.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 89.178, "latency": 43.038, "stderr": 1.391, "cost_per_test": 0.011442, "avg_input_tokens": 971.2, "avg_output_tokens": 1726.1, "avg_reasoning_tokens": 1577.4, "avg_cache_read_tokens": 571.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 43.0, "median_query_seconds": 19.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 88.978, "latency": 8.992, "stderr": 1.402, "cost_per_test": 0.022728, "avg_input_tokens": 1399.9, "avg_output_tokens": 559.2, "avg_reasoning_tokens": 365.9, "avg_cache_write_tokens": 1397.9, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 9.0, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 88.978, "latency": 16.751, "stderr": 1.402, "cost_per_test": 0.024276, "avg_input_tokens": 974.2, "avg_output_tokens": 1860.7, "avg_reasoning_tokens": 1622.4, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 16.8, "median_query_seconds": 12.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 88.978, "latency": 195.014, "stderr": 1.456, "cost_per_test": 0.030079, "avg_input_tokens": 1404.9, "avg_output_tokens": 922.2, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 195.0, "median_query_seconds": 151.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 88.377, "latency": 19.566, "stderr": 2.136, "cost_per_test": 0.01872, "avg_input_tokens": 1085.7, "avg_output_tokens": 1030.9, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 19.6, "median_query_seconds": 13.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 88.377, "latency": 24.805, "stderr": 1.435, "cost_per_test": 0.025924, "avg_input_tokens": 951.4, "avg_output_tokens": 705.6, "avg_reasoning_tokens": 645.9, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 24.8, "median_query_seconds": 21.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 88.377, "latency": 44.233, "stderr": 1.435, "cost_per_test": 0.013668, "avg_input_tokens": 1022.9, "avg_output_tokens": 1937.0, "avg_reasoning_tokens": 1827.5, "turn_metrics": { "tasks_measured": 497, "avg_turns": 1.0, "avg_query_seconds": 44.2, "median_query_seconds": 13.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 88.176, "latency": 7.866, "stderr": 1.445, "cost_per_test": 0.019743, "avg_input_tokens": 974.2, "avg_output_tokens": 2437.5, "avg_reasoning_tokens": 2260.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 7.9, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 88.176, "latency": 20.153, "stderr": 1.445, "cost_per_test": 0.084337, "avg_input_tokens": 1096.7, "avg_output_tokens": 905.2, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 20.2, "median_query_seconds": 17.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 87.776, "latency": 7.596, "stderr": 1.477, "cost_per_test": 0.040147, "avg_input_tokens": 1068.7, "avg_output_tokens": 321.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 7.6, "median_query_seconds": 7.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 87.776, "latency": 34.673, "stderr": 1.466, "cost_per_test": 0.025903, "avg_input_tokens": 976.5, "avg_output_tokens": 1276.3, "avg_reasoning_tokens": 1204.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 34.7, "median_query_seconds": 21.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 87.575, "latency": 6.594, "stderr": 1.477, "cost_per_test": 0.010596, "avg_input_tokens": 974.2, "avg_output_tokens": 1217.9, "avg_reasoning_tokens": 1034.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 6.6, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 87.575, "latency": 10.617, "stderr": 1.477, "cost_per_test": 0.01891, "avg_input_tokens": 974.2, "avg_output_tokens": 1413.4, "avg_reasoning_tokens": 1216.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 10.6, "median_query_seconds": 8.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 87.575, "latency": 21.155, "stderr": 1.477, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 87.575, "latency": 356.138, "stderr": 1.477, "cost_per_test": 0.009439, "avg_input_tokens": 971.2, "avg_output_tokens": 1436.6, "avg_reasoning_tokens": 1249.8, "avg_cache_read_tokens": 748.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 356.1, "median_query_seconds": 400.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 87.375, "latency": 34.78, "stderr": 1.487, "cost_per_test": 0.008642, "avg_input_tokens": 952.5, "avg_output_tokens": 1937.4, "avg_reasoning_tokens": 1727.9, "avg_cache_read_tokens": 710.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 34.8, "median_query_seconds": 21.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 87.174, "latency": 18.405, "stderr": 1.507, "cost_per_test": 0.009215, "avg_input_tokens": 1109.5, "avg_output_tokens": 2060.8, "avg_reasoning_tokens": 1892.2, "avg_cache_read_tokens": 845.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 18.4, "median_query_seconds": 13.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 87.174, "latency": 38.14, "stderr": 1.497, "cost_per_test": 0.006638, "avg_input_tokens": 984.9, "avg_output_tokens": 2048.4, "avg_reasoning_tokens": 1873.5, "turn_metrics": { "tasks_measured": 496, "avg_turns": 1.0, "avg_query_seconds": 38.1, "median_query_seconds": 35.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 86.974, "latency": 50.281, "stderr": 1.908, "cost_per_test": 0.047208, "avg_input_tokens": 951.4, "avg_output_tokens": 1534.5, "avg_reasoning_tokens": 1454.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 50.3, "median_query_seconds": 25.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 86.774, "latency": 1170.986, "stderr": 1.517, "cost_per_test": 0.006207, "avg_input_tokens": 943.6, "avg_output_tokens": 2329.4, "avg_reasoning_tokens": 2089.2, "avg_cache_read_tokens": 758.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 1171.0, "median_query_seconds": 982.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 86.573, "latency": 12.119, "stderr": 1.526, "cost_per_test": 0.031561, "avg_input_tokens": 1399.9, "avg_output_tokens": 981.9, "avg_cache_write_tokens": 10.8, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 12.1, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 86.573, "latency": 25.082, "stderr": 1.536, "cost_per_test": 0.039802, "avg_input_tokens": 1399.9, "avg_output_tokens": 2342.2, "avg_cache_read_tokens": 167.6, "avg_cache_write_tokens": 1230.3, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 25.1, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 86.172, "latency": 20.743, "stderr": 1.97, "cost_per_test": 0.02245, "avg_input_tokens": 1085.8, "avg_output_tokens": 1279.5, "turn_metrics": { "tasks_measured": 498, "avg_turns": 1.0, "avg_query_seconds": 20.7, "median_query_seconds": 13.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 86.172, "latency": 25.258, "stderr": 1.545, "cost_per_test": 0.016946, "avg_input_tokens": 951.4, "avg_output_tokens": 1575.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 25.3, "median_query_seconds": 19.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 85.972, "latency": 18.756, "stderr": 1.555, "cost_per_test": 0.011306, "avg_input_tokens": 951.4, "avg_output_tokens": 1011.7, "avg_reasoning_tokens": 890.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 18.8, "median_query_seconds": 11.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 85.972, "latency": 23.972, "stderr": 1.555, "cost_per_test": 0.001774, "avg_input_tokens": 970.7, "avg_output_tokens": 1183.6, "avg_cache_read_tokens": 31.0, "avg_cache_write_tokens": 937.7, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 24.0, "median_query_seconds": 16.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 85.972, "latency": 26.471, "stderr": 1.582, "cost_per_test": 0.025073, "avg_input_tokens": 1096.7, "avg_output_tokens": 1452.2, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 26.5, "median_query_seconds": 18.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 85.571, "latency": 9.721, "stderr": 1.573, "cost_per_test": 0.039418, "avg_input_tokens": 1068.7, "avg_output_tokens": 311.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 9.7, "median_query_seconds": 9.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 85.571, "latency": 24.271, "stderr": 1.609, "cost_per_test": 0.024711, "avg_input_tokens": 964.5, "avg_output_tokens": 3215.1, "avg_reasoning_tokens": 1570.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 24.3, "median_query_seconds": 13.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 85.571, "latency": 60.923, "stderr": 1.573, "cost_per_test": 0.00662, "avg_input_tokens": 984.9, "avg_output_tokens": 2594.3, "avg_reasoning_tokens": 2238.4, "turn_metrics": { "tasks_measured": 496, "avg_turns": 1.0, "avg_query_seconds": 60.9, "median_query_seconds": 52.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 85.371, "latency": 27.179, "stderr": 1.582, "cost_per_test": 0.013969, "avg_input_tokens": 985.1, "avg_output_tokens": 4492.1, "avg_reasoning_tokens": 4127.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 27.2, "median_query_seconds": 15.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark": { "accuracy": 85.371, "latency": 34.617, "stderr": 1.582, "cost_per_test": 0.001328, "avg_input_tokens": 1007.6, "avg_output_tokens": 1965.6, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 34.6, "median_query_seconds": 31.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 85.371, "latency": 41.629, "stderr": 1.582, "cost_per_test": 0.016284, "avg_input_tokens": 1036.9, "avg_output_tokens": 1046.2, "avg_reasoning_tokens": 854.8, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 41.6, "median_query_seconds": 20.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 85.17, "latency": 14.017, "stderr": 1.591, "cost_per_test": 0.007963, "avg_input_tokens": 951.4, "avg_output_tokens": 757.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 14.0, "median_query_seconds": 9.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 84.569, "latency": 35.176, "stderr": 1.729, "cost_per_test": 0.00849, "avg_input_tokens": 1045.5, "avg_output_tokens": 2054.6, "avg_reasoning_tokens": 1983.1, "avg_cache_read_tokens": 804.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 35.2, "median_query_seconds": 10.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 84.569, "latency": 36.864, "stderr": 1.617, "cost_per_test": 0.000892, "avg_input_tokens": 966.5, "avg_output_tokens": 3423.0, "avg_cache_read_tokens": 601.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 36.9, "median_query_seconds": 7.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 84.569, "latency": 70.384, "stderr": 1.617, "cost_per_test": 0.011206, "avg_input_tokens": 958.9, "avg_output_tokens": 2721.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 746.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 498, "avg_turns": 1.0, "avg_query_seconds": 70.4, "median_query_seconds": 43.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3": { "accuracy": 84.569, "latency": 82.002, "stderr": 1.617, "cost_per_test": 0.020607, "avg_input_tokens": 966.6, "avg_output_tokens": 4484.0, "avg_reasoning_tokens": 4287.6, "avg_cache_read_tokens": 380.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 82.0, "median_query_seconds": 12.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 84.569, "latency": 657.825, "stderr": 1.617, "cost_per_test": 0.014343, "avg_input_tokens": 1032.5, "avg_output_tokens": 3562.9, "avg_reasoning_tokens": 3469.8, "avg_cache_read_tokens": 884.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 657.8, "median_query_seconds": 608.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 84.168, "latency": 43.916, "stderr": 1.634, "cost_per_test": 0.011096, "avg_input_tokens": 966.5, "avg_output_tokens": 2338.2, "avg_reasoning_tokens": 2141.6, "avg_cache_read_tokens": 478.5, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 43.9, "median_query_seconds": 21.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 83.968, "latency": 6.126, "stderr": 1.642, "cost_per_test": 0.006548, "avg_input_tokens": 951.4, "avg_output_tokens": 387.1, "avg_reasoning_tokens": 332.6, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 6.1, "median_query_seconds": 3.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 83.936, "latency": 32.773, "stderr": 1.645, "cost_per_test": 0.006152, "avg_input_tokens": 959.5, "avg_output_tokens": 1824.0, "avg_reasoning_tokens": 1565.6, "avg_cache_read_tokens": 805.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 498, "avg_turns": 1.0, "avg_query_seconds": 32.8, "median_query_seconds": 17.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 83.367, "latency": 6.419, "stderr": 1.667, "cost_per_test": 0.008539, "avg_input_tokens": 1061.5, "avg_output_tokens": 1340.8, "avg_reasoning_tokens": 1151.3, "avg_cache_read_tokens": 904.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 6.4, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 83.367, "latency": 16.678, "stderr": 1.667, "cost_per_test": 0.002161, "avg_input_tokens": 1032.5, "avg_output_tokens": 1577.2, "avg_reasoning_tokens": 1521.1, "avg_cache_read_tokens": 881.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 16.7, "median_query_seconds": 4.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4-0709": { "accuracy": 83.367, "latency": 57.631, "stderr": 1.667, "cost_per_test": 0.029648, "avg_input_tokens": 937.6, "avg_output_tokens": 1789.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 57.6, "median_query_seconds": 30.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 83.166, "latency": 73.251, "stderr": 1.683, "cost_per_test": 0.00732, "avg_input_tokens": 959.3, "avg_output_tokens": 2182.6, "avg_cache_read_tokens": 761.8, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 73.3, "median_query_seconds": 61.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 82.966, "latency": 4.243, "stderr": 1.683, "cost_per_test": 0.003441, "avg_input_tokens": 974.2, "avg_output_tokens": 1259.4, "avg_reasoning_tokens": 1076.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 4.2, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 82.766, "latency": 15.843, "stderr": 1.698, "cost_per_test": 0.002589, "avg_input_tokens": 951.4, "avg_output_tokens": 1998.7, "avg_reasoning_tokens": 1934.6, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 15.8, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 82.565, "latency": 15.171, "stderr": 1.698, "cost_per_test": 0.00737, "avg_input_tokens": 980.3, "avg_output_tokens": 614.4, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 15.2, "median_query_seconds": 14.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 82.164, "latency": 26.244, "stderr": 1.714, "cost_per_test": 0.005272, "avg_input_tokens": 958.9, "avg_output_tokens": 1688.6, "avg_cache_read_tokens": 737.2, "turn_metrics": { "tasks_measured": 498, "avg_turns": 1.0, "avg_query_seconds": 26.2, "median_query_seconds": 19.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 82.164, "latency": 51.445, "stderr": 1.714, "cost_per_test": 0.025216, "avg_input_tokens": 959.4, "avg_output_tokens": 2569.6, "avg_cache_read_tokens": 810.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 51.4, "median_query_seconds": 15.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 81.764, "latency": 6.827, "stderr": 1.729, "cost_per_test": 0.00079, "avg_input_tokens": 974.2, "avg_output_tokens": 1730.4, "avg_reasoning_tokens": 1484.3, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 6.8, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 81.563, "latency": 38.216, "stderr": 1.736, "cost_per_test": 0.033281, "avg_input_tokens": 1096.7, "avg_output_tokens": 1999.4, "turn_metrics": { "tasks_measured": 498, "avg_turns": 1.0, "avg_query_seconds": 38.2, "median_query_seconds": 23.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 81.563, "latency": 79.18, "stderr": 1.736, "cost_per_test": 0.017179, "avg_input_tokens": 964.8, "avg_output_tokens": 2670.3, "avg_reasoning_tokens": 2170.6, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 495, "avg_turns": 1.0, "avg_query_seconds": 79.2, "median_query_seconds": 63.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 81.162, "latency": 14.651, "stderr": 1.75, "cost_per_test": 0.00065, "avg_input_tokens": 1086.6, "avg_output_tokens": 1129.4, "avg_reasoning_tokens": 1009.9, "avg_cache_read_tokens": 878.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 14.7, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 80.962, "latency": 16.114, "stderr": 1.758, "cost_per_test": null, "avg_input_tokens": 992.3, "avg_output_tokens": 1552.8, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 16.1, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 80.962, "latency": 19.121, "stderr": 1.758, "cost_per_test": 0.086531, "avg_input_tokens": 951.4, "avg_output_tokens": 1204.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 19.1, "median_query_seconds": 15.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 80.762, "latency": 10.531, "stderr": 1.765, "cost_per_test": 0.008185, "avg_input_tokens": 1068.7, "avg_output_tokens": 331.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 10.5, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 80.561, "latency": 15.236, "stderr": 1.785, "cost_per_test": 0.003351, "avg_input_tokens": 951.4, "avg_output_tokens": 1556.4, "avg_reasoning_tokens": 1470.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 15.2, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 80.561, "latency": 52.89, "stderr": 1.772, "cost_per_test": 0.003557, "avg_input_tokens": 959.4, "avg_output_tokens": 2890.1, "avg_cache_read_tokens": 830.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 52.9, "median_query_seconds": 17.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 80.561, "latency": 61.44, "stderr": 1.772, "cost_per_test": 0.000613, "avg_input_tokens": 953.5, "avg_output_tokens": 1304.5, "avg_reasoning_tokens": 1232.8, "avg_cache_read_tokens": 799.9, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 61.4, "median_query_seconds": 48.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 80.16, "latency": 23.038, "stderr": 1.792, "cost_per_test": 0.002277, "avg_input_tokens": 1108.7, "avg_output_tokens": 847.1, "avg_cache_read_tokens": 877.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 23.0, "median_query_seconds": 16.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-4.5": { "accuracy": 80.16, "latency": 74.109, "stderr": 1.785, "cost_per_test": 0.005902, "avg_input_tokens": 959.5, "avg_output_tokens": 2421.3, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 74.1, "median_query_seconds": 64.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 79.96, "latency": 32.952, "stderr": 2.116, "cost_per_test": 0.000991, "avg_input_tokens": 1205.8, "avg_output_tokens": 1027.5, "avg_reasoning_tokens": 880.2, "avg_cache_read_tokens": 991.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 33.0, "median_query_seconds": 20.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 79.76, "latency": 24.985, "stderr": 1.799, "cost_per_test": 0.029247, "avg_input_tokens": 1096.7, "avg_output_tokens": 1730.5, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 25.0, "median_query_seconds": 17.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 79.559, "latency": 85.402, "stderr": 1.812, "cost_per_test": 0.001517, "avg_input_tokens": 958.8, "avg_output_tokens": 2094.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 85.4, "median_query_seconds": 54.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 79.359, "latency": 5.223, "stderr": 1.812, "cost_per_test": 0.000995, "avg_input_tokens": 980.3, "avg_output_tokens": 1033.0, "avg_reasoning_tokens": 752.7, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 5.2, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 79.158, "latency": 51.886, "stderr": 1.831, "cost_per_test": 0.000163, "avg_input_tokens": 953.5, "avg_output_tokens": 303.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 916.6, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 51.9, "median_query_seconds": 40.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 78.958, "latency": 3.994, "stderr": 1.825, "cost_per_test": 0.001108, "avg_input_tokens": 1047.6, "avg_output_tokens": 1796.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 4.0, "median_query_seconds": 2.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 78.958, "latency": 36.885, "stderr": 1.825, "cost_per_test": 0.00145, "avg_input_tokens": 984.9, "avg_output_tokens": 3379.1, "avg_reasoning_tokens": 3027.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 496, "avg_turns": 1.0, "avg_query_seconds": 36.9, "median_query_seconds": 31.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 78.758, "latency": 8.332, "stderr": 1.831, "cost_per_test": 0.001001, "avg_input_tokens": 980.3, "avg_output_tokens": 1067.3, "avg_reasoning_tokens": 784.6, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 8.3, "median_query_seconds": 7.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 78.758, "latency": 20.433, "stderr": 1.837, "cost_per_test": 0.000429, "avg_input_tokens": 1201.8, "avg_output_tokens": 1431.9, "avg_reasoning_tokens": 1336.9, "avg_cache_read_tokens": 1018.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 20.4, "median_query_seconds": 13.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3-max": { "accuracy": 78.557, "latency": 0.0, "stderr": 1.837, "cost_per_test": 0.003992, "avg_input_tokens": 963.0, "avg_output_tokens": 472.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 78.557, "latency": 26.721, "stderr": 1.837, "cost_per_test": 0.007942, "avg_input_tokens": 1026.7, "avg_output_tokens": 2476.2, "avg_reasoning_tokens": 2365.5, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 26.7, "median_query_seconds": 9.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 78.557, "latency": 34.792, "stderr": 1.837, "cost_per_test": 0.004204, "avg_input_tokens": 963.0, "avg_output_tokens": 508.1, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 495, "avg_turns": 1.0, "avg_query_seconds": 34.8, "median_query_seconds": 28.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-4.7": { "accuracy": 77.956, "latency": 77.484, "stderr": 1.862, "cost_per_test": 0.005365, "avg_input_tokens": 959.5, "avg_output_tokens": 2351.2, "avg_cache_read_tokens": 693.4, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 77.5, "median_query_seconds": 37.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 77.154, "latency": 6.058, "stderr": 1.879, "cost_per_test": 0.007848, "avg_input_tokens": 1068.7, "avg_output_tokens": 309.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 6.1, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 76.553, "latency": 17.808, "stderr": 2.072, "cost_per_test": 0.008096, "avg_input_tokens": 952.5, "avg_output_tokens": 1261.1, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 17.8, "median_query_seconds": 11.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 76.353, "latency": 9.977, "stderr": 2.169, "cost_per_test": 0.001026, "avg_input_tokens": 979.3, "avg_output_tokens": 357.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 10.0, "median_query_seconds": 9.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 76.353, "latency": 18.234, "stderr": 1.902, "cost_per_test": 0.013185, "avg_input_tokens": 952.4, "avg_output_tokens": 1410.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 18.2, "median_query_seconds": 3.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.6": { "accuracy": 76.353, "latency": 35.981, "stderr": 1.902, "cost_per_test": 0.005123, "avg_input_tokens": 961.2, "avg_output_tokens": 2066.4, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 36.0, "median_query_seconds": 25.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 76.152, "latency": 6.094, "stderr": 1.908, "cost_per_test": 0.007086, "avg_input_tokens": 1068.7, "avg_output_tokens": 258.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 6.1, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 75.752, "latency": 24.442, "stderr": 1.919, "cost_per_test": 0.000964, "avg_input_tokens": 952.5, "avg_output_tokens": 118.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 24.4, "median_query_seconds": 19.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 75.551, "latency": 4.735, "stderr": 1.924, "cost_per_test": 0.000208, "avg_input_tokens": 998.7, "avg_output_tokens": 811.6, "avg_cache_read_tokens": 763.8, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 3.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 75.551, "latency": 5.384, "stderr": 1.924, "cost_per_test": 0.000764, "avg_input_tokens": 937.6, "avg_output_tokens": 964.9, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 5.4, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 75.351, "latency": 15.556, "stderr": 1.929, "cost_per_test": 0.00387, "avg_input_tokens": 951.4, "avg_output_tokens": 1816.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 15.6, "median_query_seconds": 12.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 74.95, "latency": 8.91, "stderr": 1.94, "cost_per_test": 0.001047, "avg_input_tokens": 937.6, "avg_output_tokens": 1531.3, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 8.9, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 74.549, "latency": 18.668, "stderr": 2.231, "cost_per_test": 0.012033, "avg_input_tokens": 1096.7, "avg_output_tokens": 2187.3, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 18.7, "median_query_seconds": 11.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-3": { "accuracy": 74.148, "latency": 13.985, "stderr": 1.96, "cost_per_test": 0.013679, "avg_input_tokens": 936.6, "avg_output_tokens": 724.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 14.0, "median_query_seconds": 11.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 73.747, "latency": 4.691, "stderr": 1.97, "cost_per_test": 0.000595, "avg_input_tokens": 955.5, "avg_output_tokens": 437.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 73.747, "latency": 9.06, "stderr": 1.97, "cost_per_test": 0.001606, "avg_input_tokens": 967.9, "avg_output_tokens": 212.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 9.1, "median_query_seconds": 7.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 73.747, "latency": 40.819, "stderr": 1.989, "cost_per_test": 0.002607, "avg_input_tokens": 953.7, "avg_output_tokens": 1878.7, "avg_cache_read_tokens": 16.0, "avg_cache_write_tokens": 937.7, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 40.8, "median_query_seconds": 25.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 73.547, "latency": 11.216, "stderr": 1.975, "cost_per_test": 0.004966, "avg_input_tokens": 951.4, "avg_output_tokens": 890.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 11.2, "median_query_seconds": 9.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 73.347, "latency": 2.348, "stderr": 1.979, "cost_per_test": 0.000367, "avg_input_tokens": 980.3, "avg_output_tokens": 673.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 2.3, "median_query_seconds": 2.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 73.146, "latency": 52.443, "stderr": 1.984, "cost_per_test": 0.029496, "avg_input_tokens": 992.1, "avg_output_tokens": 3734.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 484, "avg_turns": 1.0, "avg_query_seconds": 52.4, "median_query_seconds": 48.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 72.946, "latency": 2.103, "stderr": 1.989, "cost_per_test": 0.000151, "avg_input_tokens": 1098.6, "avg_output_tokens": 126.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 877.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 2.1, "median_query_seconds": 1.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 72.345, "latency": 40.377, "stderr": 2.028, "cost_per_test": 0.001476, "avg_input_tokens": 962.8, "avg_output_tokens": 1436.2, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 40.4, "median_query_seconds": 24.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 71.944, "latency": 7.43, "stderr": 2.011, "cost_per_test": 0.000237, "avg_input_tokens": 980.3, "avg_output_tokens": 1352.0, "avg_reasoning_tokens": 1003.8, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 7.4, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 71.543, "latency": 2.256, "stderr": 2.02, "cost_per_test": 0.00022, "avg_input_tokens": 980.3, "avg_output_tokens": 303.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 2.3, "median_query_seconds": 1.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 71.543, "latency": 5.458, "stderr": 2.028, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 71.142, "latency": 6.384, "stderr": 2.056, "cost_per_test": 0.005199, "avg_input_tokens": 952.4, "avg_output_tokens": 281.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 6.4, "median_query_seconds": 4.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 70.341, "latency": 2.096, "stderr": 2.045, "cost_per_test": 0.001742, "avg_input_tokens": 980.3, "avg_output_tokens": 103.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 2.1, "median_query_seconds": 1.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-2-1212": { "accuracy": 70.341, "latency": 4.414, "stderr": 2.045, "cost_per_test": 0.003941, "avg_input_tokens": 945.6, "avg_output_tokens": 205.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 4.4, "median_query_seconds": 3.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 70.341, "latency": 11.205, "stderr": 2.045, "cost_per_test": 0.000936, "avg_input_tokens": 952.5, "avg_output_tokens": 87.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 11.2, "median_query_seconds": 8.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 70.14, "latency": 22.523, "stderr": 2.049, "cost_per_test": 0.009835, "avg_input_tokens": 979.3, "avg_output_tokens": 1575.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 22.5, "median_query_seconds": 12.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 69.756, "latency": 2.738, "stderr": 1.435, "cost_per_test": 0.000827, "avg_input_tokens": 1057.6, "avg_output_tokens": 1231.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 2.1, "avg_query_seconds": 2.7, "median_query_seconds": 1.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 69.539, "latency": 38.873, "stderr": 2.064, "cost_per_test": 0.00379, "avg_input_tokens": 973.7, "avg_output_tokens": 1432.1, "avg_cache_read_tokens": 36.0, "avg_cache_write_tokens": 937.7, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 38.9, "median_query_seconds": 29.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 69.138, "latency": 20.18, "stderr": 2.068, "cost_per_test": 0.009987, "avg_input_tokens": 951.4, "avg_output_tokens": 2032.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 20.2, "median_query_seconds": 15.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 68.737, "latency": 2.836, "stderr": 2.075, "cost_per_test": 0.00094, "avg_input_tokens": 952.4, "avg_output_tokens": 349.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 2.8, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 67.535, "latency": 5.941, "stderr": 2.096, "cost_per_test": 0.005383, "avg_input_tokens": 952.4, "avg_output_tokens": 300.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 5.9, "median_query_seconds": 5.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 67.134, "latency": 10.104, "stderr": 2.103, "cost_per_test": 0.00128, "avg_input_tokens": 1019.4, "avg_output_tokens": 1879.2, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 10.1, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 66.533, "latency": 26.197, "stderr": 2.116, "cost_per_test": null, "avg_input_tokens": 979.1, "avg_output_tokens": 1102.5, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 26.2, "median_query_seconds": 20.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 66.333, "latency": 32.43, "stderr": 2.133, "cost_per_test": 0.000828, "avg_input_tokens": 1001.8, "avg_output_tokens": 1794.1, "avg_reasoning_tokens": 1578.7, "avg_cache_read_tokens": 896.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 32.4, "median_query_seconds": 20.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 66.132, "latency": 5.847, "stderr": 2.172, "cost_per_test": 0.00023, "avg_input_tokens": 951.8, "avg_output_tokens": 456.0, "avg_reasoning_tokens": 349.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 5.8, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 65.13, "latency": 22.338, "stderr": 2.133, "cost_per_test": 0.001592, "avg_input_tokens": 951.4, "avg_output_tokens": 3860.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 22.3, "median_query_seconds": 20.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 63.727, "latency": 4.929, "stderr": 2.152, "cost_per_test": 0.003555, "avg_input_tokens": 1079.0, "avg_output_tokens": 232.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 4.9, "median_query_seconds": 3.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 62.124, "latency": 7.621, "stderr": 2.178, "cost_per_test": 0.005916, "avg_input_tokens": 970.8, "avg_output_tokens": 348.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 7.6, "median_query_seconds": 5.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 61.723, "latency": 15.884, "stderr": 2.21, "cost_per_test": null, "avg_input_tokens": 979.1, "avg_output_tokens": 672.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 15.9, "median_query_seconds": 14.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 61.523, "latency": 4.873, "stderr": 2.178, "cost_per_test": 0.001224, "avg_input_tokens": 986.1, "avg_output_tokens": 404.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 4.9, "median_query_seconds": 4.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 58.918, "latency": 31.595, "stderr": 2.237, "cost_per_test": 0.000303, "avg_input_tokens": 1038.8, "avg_output_tokens": 1218.4, "avg_reasoning_tokens": 1011.5, "avg_cache_read_tokens": 885.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 31.6, "median_query_seconds": 25.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 58.717, "latency": 24.252, "stderr": 2.214, "cost_per_test": 0.000851, "avg_input_tokens": 1019.4, "avg_output_tokens": 2599.1, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 24.3, "median_query_seconds": 22.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 58.517, "latency": 3.092, "stderr": 2.206, "cost_per_test": 0.000351, "avg_input_tokens": 979.3, "avg_output_tokens": 257.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 3.1, "median_query_seconds": 2.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 58.317, "latency": 5.723, "stderr": 2.207, "cost_per_test": 0.001843, "avg_input_tokens": 1068.7, "avg_output_tokens": 246.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 5.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 57.515, "latency": 10.587, "stderr": 2.213, "cost_per_test": 0.002458, "avg_input_tokens": 979.3, "avg_output_tokens": 1312.5, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 10.6, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 57.315, "latency": 4.895, "stderr": 2.214, "cost_per_test": 0.00041, "avg_input_tokens": 955.5, "avg_output_tokens": 403.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 4.9, "median_query_seconds": 4.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 56.513, "latency": 1.282, "stderr": 2.219, "cost_per_test": 0.000123, "avg_input_tokens": 980.3, "avg_output_tokens": 166.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 1.3, "median_query_seconds": 1.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 55.311, "latency": 2.479, "stderr": 2.227, "cost_per_test": 0.000136, "avg_input_tokens": 979.3, "avg_output_tokens": 207.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 2.5, "median_query_seconds": 1.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 52.906, "latency": 1.806, "stderr": 2.237, "cost_per_test": 0.000241, "avg_input_tokens": 952.4, "avg_output_tokens": 364.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 1.8, "median_query_seconds": 1.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 50.701, "latency": 4.171, "stderr": 2.238, "cost_per_test": 0.000319, "avg_input_tokens": 952.4, "avg_output_tokens": 293.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 4.2, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 47.295, "latency": 4.55, "stderr": 2.235, "cost_per_test": 0.003064, "avg_input_tokens": 1040.1, "avg_output_tokens": 123.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 4.5, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "cohere/command-r-plus": { "accuracy": 42.084, "latency": 2.517, "stderr": 2.209, "cost_per_test": 0.003737, "avg_input_tokens": 970.8, "avg_output_tokens": 131.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 2.5, "median_query_seconds": 1.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 35.671, "latency": 1.912, "stderr": 2.144, "cost_per_test": 0.000301, "avg_input_tokens": 1040.1, "avg_output_tokens": 231.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 499, "avg_turns": 1.0, "avg_query_seconds": 1.9, "median_query_seconds": 0.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "physics": { "anthropic/claude-fable-5-1": { "accuracy": 96.69, "latency": 17.878, "stderr": 0.496, "cost_per_test": 0.086595, "avg_input_tokens": 1578.8, "avg_output_tokens": 1337.7, "avg_cache_read_tokens": 1.1, "avg_cache_write_tokens": 1573.7, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 17.9, "median_query_seconds": 7.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 95.843, "latency": 15.9, "stderr": 0.554, "cost_per_test": 0.071684, "avg_input_tokens": 1576.8, "avg_output_tokens": 1116.7, "avg_cache_write_tokens": 32.1, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 15.9, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 94.765, "latency": 7.952, "stderr": 0.618, "cost_per_test": 0.024694, "avg_input_tokens": 1230.3, "avg_output_tokens": 3046.5, "avg_reasoning_tokens": 2726.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 8.0, "median_query_seconds": 3.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 94.765, "latency": 8.999, "stderr": 0.618, "cost_per_test": 0.025532, "avg_input_tokens": 1576.8, "avg_output_tokens": 627.2, "avg_reasoning_tokens": 386.1, "avg_cache_write_tokens": 1574.8, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 9.0, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 94.688, "latency": 35.667, "stderr": 0.622, "cost_per_test": 0.038713, "avg_input_tokens": 1230.3, "avg_output_tokens": 3021.0, "avg_reasoning_tokens": 2602.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 35.7, "median_query_seconds": 19.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 94.303, "latency": 4.242, "stderr": 0.643, "cost_per_test": 0.012698, "avg_input_tokens": 1230.3, "avg_output_tokens": 1447.1, "avg_reasoning_tokens": 1151.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 4.2, "median_query_seconds": 2.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 94.149, "latency": 61.982, "stderr": 1.175, "cost_per_test": 0.066938, "avg_input_tokens": 1284.8, "avg_output_tokens": 4205.5, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 62.0, "median_query_seconds": 12.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 94.072, "latency": 58.125, "stderr": 0.663, "cost_per_test": 0.029412, "avg_input_tokens": 1245.3, "avg_output_tokens": 2243.4, "avg_reasoning_tokens": 1864.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 58.1, "median_query_seconds": 33.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 93.841, "latency": 21.651, "stderr": 0.69, "cost_per_test": 0.011016, "avg_input_tokens": 1293.4, "avg_output_tokens": 2483.1, "avg_reasoning_tokens": 2233.3, "avg_cache_read_tokens": 1049.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 21.7, "median_query_seconds": 12.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 93.687, "latency": 29.722, "stderr": 0.675, "cost_per_test": 0.046396, "avg_input_tokens": 1232.8, "avg_output_tokens": 3660.9, "avg_reasoning_tokens": 3143.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 29.7, "median_query_seconds": 20.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 93.533, "latency": 23.621, "stderr": 0.682, "cost_per_test": 0.061096, "avg_input_tokens": 1576.8, "avg_output_tokens": 2127.2, "avg_cache_write_tokens": 24.6, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 23.6, "median_query_seconds": 8.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 93.533, "latency": 37.31, "stderr": 0.682, "cost_per_test": 0.020998, "avg_input_tokens": 1243.0, "avg_output_tokens": 2385.4, "avg_reasoning_tokens": 1942.1, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 37.3, "median_query_seconds": 28.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 93.457, "latency": 8.325, "stderr": 0.686, "cost_per_test": 0.015989, "avg_input_tokens": 1230.3, "avg_output_tokens": 1885.8, "avg_reasoning_tokens": 1563.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 8.3, "median_query_seconds": 5.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 93.457, "latency": 561.23, "stderr": 0.701, "cost_per_test": 0.038088, "avg_input_tokens": 1224.0, "avg_output_tokens": 9535.8, "avg_reasoning_tokens": 9311.3, "avg_cache_read_tokens": 1011.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 561.2, "median_query_seconds": 584.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 93.38, "latency": 42.839, "stderr": 0.69, "cost_per_test": 0.023862, "avg_input_tokens": 1230.3, "avg_output_tokens": 7748.8, "avg_reasoning_tokens": 7301.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 42.8, "median_query_seconds": 30.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 93.38, "latency": 47.748, "stderr": 0.69, "cost_per_test": 0.010858, "avg_input_tokens": 1136.3, "avg_output_tokens": 2457.8, "avg_reasoning_tokens": 2189.4, "avg_cache_read_tokens": 916.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1297, "avg_turns": 1.0, "avg_query_seconds": 47.7, "median_query_seconds": 22.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 93.38, "latency": 90.591, "stderr": 0.69, "cost_per_test": 0.053061, "avg_input_tokens": 1581.8, "avg_output_tokens": 2933.2, "avg_cache_write_tokens": 23.4, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 90.6, "median_query_seconds": 67.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 93.303, "latency": 60.36, "stderr": 0.694, "cost_per_test": 0.01678, "avg_input_tokens": 1281.0, "avg_output_tokens": 2667.8, "avg_reasoning_tokens": 2431.2, "avg_cache_read_tokens": 1022.4, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 60.4, "median_query_seconds": 15.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 92.764, "latency": 13.717, "stderr": 0.719, "cost_per_test": 0.032534, "avg_input_tokens": 1180.9, "avg_output_tokens": 887.7, "avg_reasoning_tokens": 755.4, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 13.7, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 92.764, "latency": 55.516, "stderr": 0.719, "cost_per_test": 0.010983, "avg_input_tokens": 1243.0, "avg_output_tokens": 3453.7, "avg_reasoning_tokens": 3112.4, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 55.5, "median_query_seconds": 46.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 92.764, "latency": 81.557, "stderr": 0.719, "cost_per_test": 0.010281, "avg_input_tokens": 1243.0, "avg_output_tokens": 4076.6, "avg_reasoning_tokens": 3551.3, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 81.6, "median_query_seconds": 64.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark": { "accuracy": 92.687, "latency": 50.858, "stderr": 0.722, "cost_per_test": 0.001656, "avg_input_tokens": 1191.4, "avg_output_tokens": 2466.8, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 50.9, "median_query_seconds": 38.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 92.61, "latency": 47.586, "stderr": 0.726, "cost_per_test": 0.02405, "avg_input_tokens": 1230.7, "avg_output_tokens": 1556.5, "avg_reasoning_tokens": 1377.9, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 47.6, "median_query_seconds": 15.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 92.61, "latency": 64.215, "stderr": 1.234, "cost_per_test": 0.08129, "avg_input_tokens": 1284.8, "avg_output_tokens": 5162.4, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 64.2, "median_query_seconds": 14.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 92.456, "latency": 71.374, "stderr": 0.733, "cost_per_test": 0.014755, "avg_input_tokens": 1237.0, "avg_output_tokens": 3639.7, "avg_reasoning_tokens": 3481.5, "avg_cache_read_tokens": 1012.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 71.4, "median_query_seconds": 13.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 92.379, "latency": 17.441, "stderr": 0.736, "cost_per_test": 0.002804, "avg_input_tokens": 1224.0, "avg_output_tokens": 2046.1, "avg_reasoning_tokens": 1907.5, "avg_cache_read_tokens": 1021.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 17.4, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.1": { "accuracy": 92.302, "latency": 117.665, "stderr": 0.74, "cost_per_test": 0.018672, "avg_input_tokens": 1153.4, "avg_output_tokens": 5712.0, "avg_reasoning_tokens": 5398.4, "avg_cache_read_tokens": 950.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 117.7, "median_query_seconds": 43.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.3": { "accuracy": 92.302, "latency": 284.034, "stderr": 0.74, "cost_per_test": 0.011467, "avg_input_tokens": 1141.4, "avg_output_tokens": 4399.9, "avg_reasoning_tokens": 4113.8, "avg_cache_read_tokens": 913.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 284.0, "median_query_seconds": 270.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 92.225, "latency": 46.02, "stderr": 0.743, "cost_per_test": null, "avg_input_tokens": 1236.5, "avg_output_tokens": 4396.5, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 46.0, "median_query_seconds": 12.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "zai/glm-5.2": { "accuracy": 92.225, "latency": 63.91, "stderr": 0.743, "cost_per_test": 0.017374, "avg_input_tokens": 1160.4, "avg_output_tokens": 3775.8, "avg_reasoning_tokens": 3424.1, "avg_cache_read_tokens": 757.5, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 63.9, "median_query_seconds": 27.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 92.225, "latency": 65.578, "stderr": 0.743, "cost_per_test": 0.006179, "avg_input_tokens": 1157.0, "avg_output_tokens": 5060.2, "avg_cache_read_tokens": 1000.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 65.6, "median_query_seconds": 12.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.6": { "accuracy": 92.148, "latency": 47.373, "stderr": 0.746, "cost_per_test": 0.016782, "avg_input_tokens": 1179.7, "avg_output_tokens": 2606.1, "avg_reasoning_tokens": 2418.4, "avg_cache_read_tokens": 809.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 47.4, "median_query_seconds": 18.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 92.148, "latency": 52.003, "stderr": 0.746, "cost_per_test": 0.014614, "avg_input_tokens": 1160.4, "avg_output_tokens": 3121.7, "avg_reasoning_tokens": 2906.6, "avg_cache_read_tokens": 654.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 52.0, "median_query_seconds": 9.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 92.071, "latency": 186.541, "stderr": 0.753, "cost_per_test": 0.022914, "avg_input_tokens": 1152.8, "avg_output_tokens": 5645.3, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 762.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 186.5, "median_query_seconds": 71.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 91.994, "latency": 10.656, "stderr": 0.753, "cost_per_test": 0.001305, "avg_input_tokens": 1230.3, "avg_output_tokens": 2954.3, "avg_reasoning_tokens": 2533.1, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 10.7, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling": { "accuracy": 91.994, "latency": 66.021, "stderr": 0.753, "cost_per_test": 0.041044, "avg_input_tokens": 1157.0, "avg_output_tokens": 4225.3, "avg_cache_read_tokens": 946.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 66.0, "median_query_seconds": 13.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.5": { "accuracy": 91.994, "latency": 257.307, "stderr": 0.753, "cost_per_test": 0.017827, "avg_input_tokens": 1179.7, "avg_output_tokens": 2798.9, "avg_reasoning_tokens": 2543.9, "avg_cache_read_tokens": 883.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 257.3, "median_query_seconds": 246.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 91.84, "latency": 41.786, "stderr": 0.76, "cost_per_test": 0.001007, "avg_input_tokens": 1160.4, "avg_output_tokens": 3826.7, "avg_cache_read_tokens": 618.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 41.8, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 91.763, "latency": 14.975, "stderr": 0.815, "cost_per_test": 0.002594, "avg_input_tokens": 1149.0, "avg_output_tokens": 1970.3, "avg_reasoning_tokens": 1822.3, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 15.0, "median_query_seconds": 6.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 91.686, "latency": 43.672, "stderr": 1.171, "cost_per_test": 0.050012, "avg_input_tokens": 1295.8, "avg_output_tokens": 3075.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 43.7, "median_query_seconds": 19.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-5-thinking": { "accuracy": 91.686, "latency": 109.966, "stderr": 0.772, "cost_per_test": 0.01562, "avg_input_tokens": 1153.2, "avg_output_tokens": 4764.3, "avg_cache_read_tokens": 928.1, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 110.0, "median_query_seconds": 65.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 91.532, "latency": 8.615, "stderr": 0.871, "cost_per_test": 0.018511, "avg_input_tokens": 1576.8, "avg_output_tokens": 840.0, "avg_cache_write_tokens": 1574.8, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 8.6, "median_query_seconds": 3.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 91.532, "latency": 40.216, "stderr": 0.772, "cost_per_test": 0.024257, "avg_input_tokens": 1149.0, "avg_output_tokens": 1156.1, "avg_reasoning_tokens": 967.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 40.2, "median_query_seconds": 34.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 91.455, "latency": 11.019, "stderr": 0.776, "cost_per_test": 0.014481, "avg_input_tokens": 1255.0, "avg_output_tokens": 2342.1, "avg_reasoning_tokens": 2103.3, "avg_cache_read_tokens": 1156.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 11.0, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 91.378, "latency": 31.736, "stderr": 0.779, "cost_per_test": 0.001891, "avg_input_tokens": 1243.0, "avg_output_tokens": 4417.9, "avg_reasoning_tokens": 3854.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 31.7, "median_query_seconds": 26.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 91.378, "latency": 34.187, "stderr": 0.779, "cost_per_test": 0.039451, "avg_input_tokens": 1149.0, "avg_output_tokens": 2674.5, "avg_reasoning_tokens": 2451.0, "avg_cache_read_tokens": 1.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 34.2, "median_query_seconds": 13.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 91.301, "latency": 56.633, "stderr": 0.782, "cost_per_test": 0.011735, "avg_input_tokens": 1152.5, "avg_output_tokens": 3813.7, "avg_cache_read_tokens": 758.5, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 56.6, "median_query_seconds": 33.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 91.301, "latency": 57.97, "stderr": 0.782, "cost_per_test": 0.009675, "avg_input_tokens": 1285.0, "avg_output_tokens": 3010.8, "avg_reasoning_tokens": 2766.3, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 58.0, "median_query_seconds": 16.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 91.224, "latency": 19.87, "stderr": 0.931, "cost_per_test": 0.041984, "avg_input_tokens": 1149.0, "avg_output_tokens": 1413.6, "avg_reasoning_tokens": 1193.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 19.9, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 91.224, "latency": 93.164, "stderr": 1.052, "cost_per_test": 0.003086, "avg_input_tokens": 1472.0, "avg_output_tokens": 3433.8, "avg_reasoning_tokens": 3169.9, "avg_cache_read_tokens": 1250.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 93.2, "median_query_seconds": 40.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 91.224, "latency": 136.647, "stderr": 0.785, "cost_per_test": 0.032391, "avg_input_tokens": 1231.0, "avg_output_tokens": 5152.3, "avg_reasoning_tokens": 4405.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 136.6, "median_query_seconds": 82.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 91.07, "latency": 7.846, "stderr": 0.791, "cost_per_test": 0.008051, "avg_input_tokens": 1149.0, "avg_output_tokens": 479.4, "avg_reasoning_tokens": 349.3, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 7.8, "median_query_seconds": 3.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 90.839, "latency": 50.009, "stderr": 0.847, "cost_per_test": 0.006439, "avg_input_tokens": 1298.9, "avg_output_tokens": 2558.6, "avg_cache_read_tokens": 1002.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 50.0, "median_query_seconds": 16.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 90.762, "latency": 57.666, "stderr": 0.818, "cost_per_test": 0.003131, "avg_input_tokens": 1229.0, "avg_output_tokens": 3250.2, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 57.7, "median_query_seconds": 33.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 90.685, "latency": 23.472, "stderr": 0.806, "cost_per_test": 0.022039, "avg_input_tokens": 1149.0, "avg_output_tokens": 2060.2, "avg_reasoning_tokens": 1776.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 23.5, "median_query_seconds": 14.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 90.685, "latency": 44.486, "stderr": 0.806, "cost_per_test": 0.027143, "avg_input_tokens": 1149.0, "avg_output_tokens": 2570.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 44.5, "median_query_seconds": 25.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 90.608, "latency": 0.0, "stderr": 0.809, "cost_per_test": 0.007626, "avg_input_tokens": 1229.0, "avg_output_tokens": 1025.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 90.531, "latency": 5.748, "stderr": 0.812, "cost_per_test": 0.005889, "avg_input_tokens": 1230.3, "avg_output_tokens": 2208.0, "avg_reasoning_tokens": 1866.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 3.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 90.531, "latency": 16.319, "stderr": 0.812, "cost_per_test": 0.003314, "avg_input_tokens": 1149.0, "avg_output_tokens": 1513.5, "avg_reasoning_tokens": 1310.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 16.3, "median_query_seconds": 6.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 90.223, "latency": 36.498, "stderr": 0.835, "cost_per_test": 0.001022, "avg_input_tokens": 1468.0, "avg_output_tokens": 3443.9, "avg_reasoning_tokens": 3202.8, "avg_cache_read_tokens": 1079.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 36.5, "median_query_seconds": 18.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-4.6": { "accuracy": 90.223, "latency": 56.515, "stderr": 0.833, "cost_per_test": 0.007233, "avg_input_tokens": 1153.4, "avg_output_tokens": 2973.1, "avg_cache_read_tokens": 850.3, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 56.5, "median_query_seconds": 28.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-0709": { "accuracy": 90.069, "latency": 120.658, "stderr": 0.83, "cost_per_test": 0.069894, "avg_input_tokens": 1135.4, "avg_output_tokens": 4432.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 120.7, "median_query_seconds": 61.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 89.915, "latency": 17.865, "stderr": 0.835, "cost_per_test": 0.013916, "avg_input_tokens": 1149.0, "avg_output_tokens": 1452.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 17.9, "median_query_seconds": 9.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 89.915, "latency": 34.366, "stderr": 0.841, "cost_per_test": 0.154028, "avg_input_tokens": 1295.8, "avg_output_tokens": 1794.5, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 34.4, "median_query_seconds": 21.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 89.915, "latency": 47.367, "stderr": 0.989, "cost_per_test": 0.282054, "avg_input_tokens": 1296.8, "avg_output_tokens": 3501.4, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 47.4, "median_query_seconds": 18.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 89.838, "latency": 7.714, "stderr": 0.902, "cost_per_test": 0.050949, "avg_input_tokens": 1267.8, "avg_output_tokens": 425.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 7.7, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 89.838, "latency": 59.061, "stderr": 0.844, "cost_per_test": 0.060425, "avg_input_tokens": 1295.8, "avg_output_tokens": 3769.2, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 59.1, "median_query_seconds": 29.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 89.761, "latency": 21.681, "stderr": 0.841, "cost_per_test": 0.009687, "avg_input_tokens": 1220.4, "avg_output_tokens": 816.2, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 21.7, "median_query_seconds": 16.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 89.53, "latency": 8.471, "stderr": 0.849, "cost_per_test": 0.001752, "avg_input_tokens": 1220.4, "avg_output_tokens": 2245.3, "avg_reasoning_tokens": 1691.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 8.5, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 89.53, "latency": 30.45, "stderr": 0.849, "cost_per_test": 0.001349, "avg_input_tokens": 1284.4, "avg_output_tokens": 2498.7, "avg_reasoning_tokens": 2330.1, "avg_cache_read_tokens": 1050.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 30.5, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 89.53, "latency": 38.631, "stderr": 1.265, "cost_per_test": 0.016036, "avg_input_tokens": 1144.0, "avg_output_tokens": 2683.6, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 38.6, "median_query_seconds": 22.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 89.53, "latency": 52.388, "stderr": 0.849, "cost_per_test": 0.003995, "avg_input_tokens": 1160.9, "avg_output_tokens": 2975.3, "avg_cache_read_tokens": 31.0, "avg_cache_write_tokens": 1127.9, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 52.4, "median_query_seconds": 28.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 89.53, "latency": 112.283, "stderr": 0.849, "cost_per_test": 0.001525, "avg_input_tokens": 1144.9, "avg_output_tokens": 3559.2, "avg_reasoning_tokens": 3393.7, "avg_cache_read_tokens": 1110.7, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 112.3, "median_query_seconds": 53.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-4.7": { "accuracy": 89.53, "latency": 135.407, "stderr": 0.852, "cost_per_test": 0.012007, "avg_input_tokens": 1153.0, "avg_output_tokens": 5390.3, "avg_cache_read_tokens": 803.8, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 135.4, "median_query_seconds": 53.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 89.453, "latency": 17.887, "stderr": 0.852, "cost_per_test": 0.005539, "avg_input_tokens": 1149.0, "avg_output_tokens": 2625.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 17.9, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 89.376, "latency": 25.989, "stderr": 0.871, "cost_per_test": 0.001273, "avg_input_tokens": 1255.8, "avg_output_tokens": 5629.7, "avg_cache_read_tokens": 994.6, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 26.0, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 88.915, "latency": 18.345, "stderr": 0.871, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 88.838, "latency": 27.838, "stderr": 0.874, "cost_per_test": 0.002409, "avg_input_tokens": 1135.4, "avg_output_tokens": 4136.3, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 27.8, "median_query_seconds": 15.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 88.684, "latency": 11.403, "stderr": 0.879, "cost_per_test": 0.001754, "avg_input_tokens": 1220.4, "avg_output_tokens": 2220.8, "avg_reasoning_tokens": 1665.7, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 11.4, "median_query_seconds": 9.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 88.376, "latency": 53.018, "stderr": 0.889, "cost_per_test": 0.00767, "avg_input_tokens": 1229.0, "avg_output_tokens": 1032.5, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 53.0, "median_query_seconds": 26.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 88.299, "latency": 31.74, "stderr": 0.945, "cost_per_test": 0.00037, "avg_input_tokens": 1145.1, "avg_output_tokens": 687.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 945.3, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 31.7, "median_query_seconds": 27.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 87.991, "latency": 10.681, "stderr": 0.902, "cost_per_test": 0.050261, "avg_input_tokens": 1267.8, "avg_output_tokens": 416.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 10.7, "median_query_seconds": 9.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 87.991, "latency": 67.562, "stderr": 0.909, "cost_per_test": 0.005466, "avg_input_tokens": 1143.9, "avg_output_tokens": 4201.8, "avg_cache_read_tokens": 16.0, "avg_cache_write_tokens": 1127.9, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 67.6, "median_query_seconds": 21.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-3": { "accuracy": 87.914, "latency": 11.498, "stderr": 0.904, "cost_per_test": 0.01956, "avg_input_tokens": 1134.4, "avg_output_tokens": 1077.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 11.5, "median_query_seconds": 9.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 87.76, "latency": 10.012, "stderr": 0.974, "cost_per_test": 0.000492, "avg_input_tokens": 1149.4, "avg_output_tokens": 1087.0, "avg_reasoning_tokens": 830.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 10.0, "median_query_seconds": 5.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 87.76, "latency": 22.351, "stderr": 0.909, "cost_per_test": 0.013089, "avg_input_tokens": 1149.0, "avg_output_tokens": 2687.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 22.4, "median_query_seconds": 13.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 87.76, "latency": 44.271, "stderr": 0.909, "cost_per_test": 0.058227, "avg_input_tokens": 1295.8, "avg_output_tokens": 3622.6, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 44.3, "median_query_seconds": 34.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 87.529, "latency": 33.158, "stderr": 0.917, "cost_per_test": 0.131873, "avg_input_tokens": 1149.0, "avg_output_tokens": 1910.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 33.2, "median_query_seconds": 20.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 87.529, "latency": 55.012, "stderr": 0.922, "cost_per_test": 0.006352, "avg_input_tokens": 1163.9, "avg_output_tokens": 2470.1, "avg_cache_read_tokens": 37.0, "avg_cache_write_tokens": 1126.9, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 55.0, "median_query_seconds": 27.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 87.375, "latency": 10.661, "stderr": 0.922, "cost_per_test": 0.005527, "avg_input_tokens": 1149.0, "avg_output_tokens": 968.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 10.7, "median_query_seconds": 8.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.5": { "accuracy": 87.144, "latency": 96.592, "stderr": 0.929, "cost_per_test": 0.013783, "avg_input_tokens": 1153.4, "avg_output_tokens": 5950.6, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 96.6, "median_query_seconds": 73.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 86.913, "latency": 18.674, "stderr": 0.945, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 86.836, "latency": 9.383, "stderr": 0.938, "cost_per_test": 0.001892, "avg_input_tokens": 1245.4, "avg_output_tokens": 3286.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 9.4, "median_query_seconds": 4.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 86.605, "latency": 5.019, "stderr": 0.945, "cost_per_test": 0.000682, "avg_input_tokens": 1139.4, "avg_output_tokens": 490.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 5.0, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 86.528, "latency": 71.364, "stderr": 0.947, "cost_per_test": 0.056352, "avg_input_tokens": 1235.8, "avg_output_tokens": 7266.5, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1283, "avg_turns": 1.0, "avg_query_seconds": 71.4, "median_query_seconds": 67.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 86.451, "latency": 8.814, "stderr": 0.95, "cost_per_test": 0.001151, "avg_input_tokens": 1135.4, "avg_output_tokens": 1621.1, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 8.8, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 86.451, "latency": 98.509, "stderr": 1.014, "cost_per_test": 0.0022, "avg_input_tokens": 1152.4, "avg_output_tokens": 3281.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 98.5, "median_query_seconds": 44.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 86.374, "latency": 14.859, "stderr": 1.213, "cost_per_test": 0.001605, "avg_input_tokens": 1223.5, "avg_output_tokens": 663.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 14.9, "median_query_seconds": 11.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 86.22, "latency": 22.957, "stderr": 1.016, "cost_per_test": 0.001517, "avg_input_tokens": 1144.0, "avg_output_tokens": 541.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 23.0, "median_query_seconds": 19.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 86.066, "latency": 10.632, "stderr": 0.961, "cost_per_test": 0.000356, "avg_input_tokens": 1220.4, "avg_output_tokens": 2882.4, "avg_reasoning_tokens": 2297.2, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 10.6, "median_query_seconds": 8.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 85.758, "latency": 10.151, "stderr": 0.97, "cost_per_test": 0.010341, "avg_input_tokens": 1267.8, "avg_output_tokens": 435.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 10.2, "median_query_seconds": 8.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 85.604, "latency": 23.06, "stderr": 0.974, "cost_per_test": 0.002917, "avg_input_tokens": 1161.7, "avg_output_tokens": 585.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 23.1, "median_query_seconds": 16.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 85.527, "latency": 22.595, "stderr": 0.978, "cost_per_test": 0.002019, "avg_input_tokens": 1153.1, "avg_output_tokens": 4904.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 22.6, "median_query_seconds": 15.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 85.45, "latency": 34.158, "stderr": 1.299, "cost_per_test": 0.026522, "avg_input_tokens": 1295.8, "avg_output_tokens": 5045.3, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 34.2, "median_query_seconds": 15.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 84.834, "latency": 8.881, "stderr": 0.995, "cost_per_test": 0.007688, "avg_input_tokens": 1150.0, "avg_output_tokens": 673.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 8.9, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 84.45, "latency": 3.727, "stderr": 1.005, "cost_per_test": 0.000677, "avg_input_tokens": 1220.4, "avg_output_tokens": 1387.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 3.7, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 84.296, "latency": 6.592, "stderr": 1.01, "cost_per_test": 0.009941, "avg_input_tokens": 1267.8, "avg_output_tokens": 409.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 6.6, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 83.911, "latency": 5.669, "stderr": 1.019, "cost_per_test": 0.000517, "avg_input_tokens": 1220.4, "avg_output_tokens": 987.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 83.603, "latency": 4.687, "stderr": 1.027, "cost_per_test": 0.001481, "avg_input_tokens": 1150.0, "avg_output_tokens": 638.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 3.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 82.602, "latency": 22.214, "stderr": 1.154, "cost_per_test": 0.00112, "avg_input_tokens": 1217.0, "avg_output_tokens": 3448.8, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 22.2, "median_query_seconds": 17.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 81.293, "latency": 4.62, "stderr": 1.084, "cost_per_test": 0.003023, "avg_input_tokens": 1220.4, "avg_output_tokens": 299.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 4.6, "median_query_seconds": 3.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 80.216, "latency": 14.915, "stderr": 1.109, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-2-1212": { "accuracy": 79.523, "latency": 13.432, "stderr": 1.12, "cost_per_test": 0.007152, "avg_input_tokens": 1217.0, "avg_output_tokens": 471.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 13.4, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 78.214, "latency": 6.246, "stderr": 1.145, "cost_per_test": 0.008225, "avg_input_tokens": 1267.8, "avg_output_tokens": 294.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 6.2, "median_query_seconds": 6.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 78.06, "latency": 4.979, "stderr": 1.148, "cost_per_test": 0.00025, "avg_input_tokens": 1296.4, "avg_output_tokens": 296.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 1051.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 5.0, "median_query_seconds": 3.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 77.213, "latency": 7.147, "stderr": 1.164, "cost_per_test": 0.001282, "avg_input_tokens": 1144.0, "avg_output_tokens": 280.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 7.1, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 76.828, "latency": 8.434, "stderr": 1.171, "cost_per_test": 0.000539, "avg_input_tokens": 1139.4, "avg_output_tokens": 565.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 8.4, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 75.905, "latency": 25.587, "stderr": 1.187, "cost_per_test": 0.004659, "avg_input_tokens": 1223.5, "avg_output_tokens": 2700.7, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 25.6, "median_query_seconds": 20.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 75.828, "latency": 38.255, "stderr": 1.188, "cost_per_test": 0.016465, "avg_input_tokens": 1223.5, "avg_output_tokens": 2803.6, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 38.3, "median_query_seconds": 22.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 75.751, "latency": 2.071, "stderr": 1.189, "cost_per_test": 0.001043, "avg_input_tokens": 1255.4, "avg_output_tokens": 1583.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 2.1, "median_query_seconds": 1.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 75.366, "latency": 12.704, "stderr": 1.196, "cost_per_test": 0.008408, "avg_input_tokens": 1150.0, "avg_output_tokens": 553.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 12.7, "median_query_seconds": 9.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 74.211, "latency": 12.097, "stderr": 1.271, "cost_per_test": 0.008214, "avg_input_tokens": 1150.0, "avg_output_tokens": 533.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 12.1, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 72.209, "latency": 66.832, "stderr": 1.342, "cost_per_test": 0.000914, "avg_input_tokens": 1276.5, "avg_output_tokens": 4198.6, "avg_reasoning_tokens": 3862.6, "avg_cache_read_tokens": 1062.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 66.8, "median_query_seconds": 33.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 72.055, "latency": 10.883, "stderr": 1.245, "cost_per_test": 0.005755, "avg_input_tokens": 1317.2, "avg_output_tokens": 520.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 10.9, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 71.44, "latency": 145.587, "stderr": 1.262, "cost_per_test": 0.002875, "avg_input_tokens": 1239.5, "avg_output_tokens": 13306.5, "avg_reasoning_tokens": 6476.6, "avg_cache_read_tokens": 1047.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 145.6, "median_query_seconds": 98.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 71.209, "latency": 2.297, "stderr": 1.256, "cost_per_test": 0.000188, "avg_input_tokens": 1220.4, "avg_output_tokens": 322.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 2.3, "median_query_seconds": 1.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 69.284, "latency": 4.451, "stderr": 1.28, "cost_per_test": 0.001542, "avg_input_tokens": 1182.5, "avg_output_tokens": 570.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 4.5, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 69.284, "latency": 15.203, "stderr": 1.298, "cost_per_test": 0.008877, "avg_input_tokens": 1233.3, "avg_output_tokens": 579.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 15.2, "median_query_seconds": 8.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 69.053, "latency": 2.658, "stderr": 1.299, "cost_per_test": 0.000363, "avg_input_tokens": 1150.0, "avg_output_tokens": 619.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 2.7, "median_query_seconds": 2.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 68.976, "latency": 7.145, "stderr": 1.283, "cost_per_test": 0.00061, "avg_input_tokens": 1223.5, "avg_output_tokens": 608.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 7.1, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 68.206, "latency": 5.861, "stderr": 1.292, "cost_per_test": 0.000262, "avg_input_tokens": 1223.5, "avg_output_tokens": 566.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 5.9, "median_query_seconds": 3.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 66.975, "latency": 18.493, "stderr": 1.328, "cost_per_test": null, "avg_input_tokens": 1175.5, "avg_output_tokens": 789.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 18.5, "median_query_seconds": 16.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 64.896, "latency": 50.202, "stderr": 1.331, "cost_per_test": null, "avg_input_tokens": 1175.5, "avg_output_tokens": 2486.9, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 50.2, "median_query_seconds": 34.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 64.434, "latency": 7.722, "stderr": 1.328, "cost_per_test": 0.000508, "avg_input_tokens": 1150.0, "avg_output_tokens": 559.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 7.7, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 61.432, "latency": 5.691, "stderr": 1.351, "cost_per_test": 0.00214, "avg_input_tokens": 1267.8, "avg_output_tokens": 281.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 5.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 52.733, "latency": 15.319, "stderr": 1.387, "cost_per_test": 0.005276, "avg_input_tokens": 1271.6, "avg_output_tokens": 341.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 15.3, "median_query_seconds": 12.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "cohere/command-r-plus": { "accuracy": 34.642, "latency": 4.669, "stderr": 1.32, "cost_per_test": 0.005526, "avg_input_tokens": 1233.3, "avg_output_tokens": 244.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 3.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 30.023, "latency": 3.412, "stderr": 1.27, "cost_per_test": 0.000426, "avg_input_tokens": 1271.6, "avg_output_tokens": 429.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 1299, "avg_turns": 1.0, "avg_query_seconds": 3.4, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "psychology": { "anthropic/claude-fable-5-1": { "accuracy": 93.734, "latency": 12.25, "stderr": 0.858, "cost_per_test": 0.057011, "avg_input_tokens": 1630.2, "avg_output_tokens": 732.9, "avg_cache_write_tokens": 1626.2, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 12.2, "median_query_seconds": 7.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 92.481, "latency": 6.875, "stderr": 0.933, "cost_per_test": 0.019329, "avg_input_tokens": 1628.2, "avg_output_tokens": 366.2, "avg_reasoning_tokens": 174.1, "avg_cache_write_tokens": 1626.2, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 6.9, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 92.231, "latency": 13.228, "stderr": 0.948, "cost_per_test": 0.04971, "avg_input_tokens": 1628.2, "avg_output_tokens": 660.9, "avg_cache_write_tokens": 153.8, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 13.2, "median_query_seconds": 9.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 91.98, "latency": 15.965, "stderr": 0.961, "cost_per_test": 0.015685, "avg_input_tokens": 1079.1, "avg_output_tokens": 1127.2, "avg_reasoning_tokens": 971.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 16.0, "median_query_seconds": 9.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 91.479, "latency": 5.243, "stderr": 0.988, "cost_per_test": 0.013425, "avg_input_tokens": 1079.1, "avg_output_tokens": 1574.1, "avg_reasoning_tokens": 1437.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 5.2, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 91.228, "latency": 37.575, "stderr": 1.045, "cost_per_test": 0.028932, "avg_input_tokens": 1633.2, "avg_output_tokens": 829.8, "avg_cache_write_tokens": 16.3, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 37.6, "median_query_seconds": 30.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 91.103, "latency": 2.915, "stderr": 1.008, "cost_per_test": 0.006702, "avg_input_tokens": 1079.1, "avg_output_tokens": 677.8, "avg_reasoning_tokens": 559.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 2.9, "median_query_seconds": 2.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 90.977, "latency": 16.858, "stderr": 1.014, "cost_per_test": 0.024357, "avg_input_tokens": 1079.1, "avg_output_tokens": 1849.9, "avg_reasoning_tokens": 1636.5, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 16.9, "median_query_seconds": 11.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 90.852, "latency": 29.715, "stderr": 1.021, "cost_per_test": 0.008823, "avg_input_tokens": 1072.6, "avg_output_tokens": 1327.9, "avg_reasoning_tokens": 1190.7, "avg_cache_read_tokens": 859.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 29.7, "median_query_seconds": 13.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 90.351, "latency": 8.646, "stderr": 1.045, "cost_per_test": 0.015991, "avg_input_tokens": 1079.1, "avg_output_tokens": 1152.7, "avg_reasoning_tokens": 972.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 8.6, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 90.226, "latency": 341.948, "stderr": 1.051, "cost_per_test": 0.00786, "avg_input_tokens": 1072.6, "avg_output_tokens": 1174.9, "avg_reasoning_tokens": 1004.0, "avg_cache_read_tokens": 889.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 341.9, "median_query_seconds": 246.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 89.975, "latency": 13.084, "stderr": 1.063, "cost_per_test": 0.031378, "avg_input_tokens": 1628.2, "avg_output_tokens": 928.6, "avg_cache_write_tokens": 16.3, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 13.1, "median_query_seconds": 7.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 89.975, "latency": 19.369, "stderr": 1.063, "cost_per_test": 0.010724, "avg_input_tokens": 1079.2, "avg_output_tokens": 1070.1, "avg_reasoning_tokens": 872.6, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 19.4, "median_query_seconds": 17.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 89.85, "latency": 18.714, "stderr": 1.767, "cost_per_test": 0.017434, "avg_input_tokens": 1162.9, "avg_output_tokens": 929.7, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 18.7, "median_query_seconds": 15.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 89.474, "latency": 5.097, "stderr": 1.086, "cost_per_test": 0.00853, "avg_input_tokens": 1079.1, "avg_output_tokens": 921.5, "avg_reasoning_tokens": 765.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 5.1, "median_query_seconds": 3.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 89.348, "latency": 16.632, "stderr": 1.092, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 89.223, "latency": 6.702, "stderr": 1.098, "cost_per_test": 0.016633, "avg_input_tokens": 1046.2, "avg_output_tokens": 380.1, "avg_reasoning_tokens": 328.8, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 6.7, "median_query_seconds": 4.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 89.223, "latency": 89.709, "stderr": 1.211, "cost_per_test": 0.009941, "avg_input_tokens": 1057.0, "avg_output_tokens": 2381.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 747.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 89.7, "median_query_seconds": 50.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 88.847, "latency": 20.086, "stderr": 1.114, "cost_per_test": 0.010053, "avg_input_tokens": 1079.1, "avg_output_tokens": 3171.3, "avg_reasoning_tokens": 2968.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 20.1, "median_query_seconds": 7.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 88.722, "latency": 7.824, "stderr": 1.125, "cost_per_test": 0.042011, "avg_input_tokens": 1145.9, "avg_output_tokens": 331.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 7.8, "median_query_seconds": 7.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 88.722, "latency": 27.27, "stderr": 1.239, "cost_per_test": 0.045438, "avg_input_tokens": 1628.2, "avg_output_tokens": 2622.3, "avg_cache_write_tokens": 1626.2, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 27.3, "median_query_seconds": 9.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 88.471, "latency": 19.271, "stderr": 1.131, "cost_per_test": 0.080786, "avg_input_tokens": 1173.9, "avg_output_tokens": 842.4, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 19.3, "median_query_seconds": 18.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 88.471, "latency": 24.149, "stderr": 1.131, "cost_per_test": 0.017305, "avg_input_tokens": 1046.2, "avg_output_tokens": 787.0, "avg_reasoning_tokens": 717.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 24.1, "median_query_seconds": 19.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 88.471, "latency": 33.739, "stderr": 1.131, "cost_per_test": 0.010432, "avg_input_tokens": 1117.2, "avg_output_tokens": 1366.3, "avg_reasoning_tokens": 1279.6, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 33.7, "median_query_seconds": 9.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k3": { "accuracy": 88.346, "latency": 25.073, "stderr": 1.136, "cost_per_test": 0.009787, "avg_input_tokens": 1135.0, "avg_output_tokens": 609.3, "avg_reasoning_tokens": 454.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 25.1, "median_query_seconds": 16.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 88.221, "latency": 9.928, "stderr": 1.141, "cost_per_test": 0.039244, "avg_input_tokens": 1145.9, "avg_output_tokens": 294.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 9.9, "median_query_seconds": 9.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 88.221, "latency": 32.451, "stderr": 1.187, "cost_per_test": 0.111833, "avg_input_tokens": 1174.9, "avg_output_tokens": 1256.1, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 32.5, "median_query_seconds": 18.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 87.97, "latency": 22.426, "stderr": 1.157, "cost_per_test": 0.001849, "avg_input_tokens": 1060.0, "avg_output_tokens": 1218.5, "avg_cache_read_tokens": 31.0, "avg_cache_write_tokens": 1027.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 22.4, "median_query_seconds": 14.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 87.97, "latency": 492.984, "stderr": 1.167, "cost_per_test": 0.009001, "avg_input_tokens": 1124.1, "avg_output_tokens": 2235.9, "avg_reasoning_tokens": 2138.9, "avg_cache_read_tokens": 1047.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 493.0, "median_query_seconds": 548.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 87.845, "latency": 24.507, "stderr": 1.192, "cost_per_test": 0.022662, "avg_input_tokens": 1173.9, "avg_output_tokens": 1276.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 24.5, "median_query_seconds": 18.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 87.845, "latency": 34.537, "stderr": 1.474, "cost_per_test": 0.032734, "avg_input_tokens": 1046.2, "avg_output_tokens": 1041.8, "avg_reasoning_tokens": 966.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 34.5, "median_query_seconds": 12.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 87.719, "latency": 384.867, "stderr": 1.162, "cost_per_test": 0.005471, "avg_input_tokens": 1031.9, "avg_output_tokens": 1974.9, "avg_reasoning_tokens": 1747.2, "avg_cache_read_tokens": 720.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 384.9, "median_query_seconds": 362.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 87.469, "latency": 22.032, "stderr": 1.172, "cost_per_test": 0.014089, "avg_input_tokens": 1046.2, "avg_output_tokens": 1278.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 22.0, "median_query_seconds": 17.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 87.469, "latency": 33.387, "stderr": 1.172, "cost_per_test": 0.006021, "avg_input_tokens": 1079.2, "avg_output_tokens": 1827.1, "avg_reasoning_tokens": 1678.3, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 33.4, "median_query_seconds": 31.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 87.343, "latency": 28.875, "stderr": 1.339, "cost_per_test": 0.021554, "avg_input_tokens": 1162.9, "avg_output_tokens": 1204.3, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 28.9, "median_query_seconds": 15.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 87.218, "latency": 15.16, "stderr": 1.187, "cost_per_test": 0.007738, "avg_input_tokens": 1201.9, "avg_output_tokens": 1712.1, "avg_reasoning_tokens": 1563.5, "avg_cache_read_tokens": 945.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 15.2, "median_query_seconds": 11.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.1": { "accuracy": 87.218, "latency": 26.174, "stderr": 1.182, "cost_per_test": 0.004468, "avg_input_tokens": 1058.1, "avg_output_tokens": 1284.5, "avg_reasoning_tokens": 1063.2, "avg_cache_read_tokens": 876.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 26.2, "median_query_seconds": 15.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 87.218, "latency": 28.213, "stderr": 1.187, "cost_per_test": 0.00544, "avg_input_tokens": 1137.1, "avg_output_tokens": 1273.3, "avg_reasoning_tokens": 1209.7, "avg_cache_read_tokens": 864.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 28.2, "median_query_seconds": 8.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 87.093, "latency": 11.907, "stderr": 1.187, "cost_per_test": 0.009152, "avg_input_tokens": 1046.2, "avg_output_tokens": 784.4, "avg_reasoning_tokens": 664.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 11.9, "median_query_seconds": 9.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 86.967, "latency": 55.554, "stderr": 1.192, "cost_per_test": 0.006254, "avg_input_tokens": 1079.3, "avg_output_tokens": 2425.8, "avg_reasoning_tokens": 2215.7, "turn_metrics": { "tasks_measured": 797, "avg_turns": 1.0, "avg_query_seconds": 55.6, "median_query_seconds": 40.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 86.717, "latency": 13.493, "stderr": 1.201, "cost_per_test": 0.000313, "avg_input_tokens": 1065.1, "avg_output_tokens": 1078.7, "avg_cache_read_tokens": 609.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 13.5, "median_query_seconds": 6.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 86.591, "latency": 27.427, "stderr": 1.211, "cost_per_test": 0.00688, "avg_input_tokens": 1044.9, "avg_output_tokens": 1524.0, "avg_reasoning_tokens": 1352.4, "avg_cache_read_tokens": 821.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 27.4, "median_query_seconds": 16.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling": { "accuracy": 86.591, "latency": 39.249, "stderr": 1.206, "cost_per_test": 0.019293, "avg_input_tokens": 1054.2, "avg_output_tokens": 1935.5, "avg_cache_read_tokens": 924.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 39.2, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 86.466, "latency": 11.696, "stderr": 1.211, "cost_per_test": 0.007081, "avg_input_tokens": 1046.2, "avg_output_tokens": 623.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 11.7, "median_query_seconds": 8.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 86.466, "latency": 31.285, "stderr": 1.211, "cost_per_test": 0.001246, "avg_input_tokens": 1099.9, "avg_output_tokens": 1805.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 31.3, "median_query_seconds": 28.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 86.341, "latency": 4.315, "stderr": 1.216, "cost_per_test": 0.004611, "avg_input_tokens": 1046.2, "avg_output_tokens": 209.9, "avg_reasoning_tokens": 161.2, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 4.3, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 86.216, "latency": 30.286, "stderr": 1.22, "cost_per_test": 0.007257, "avg_input_tokens": 1065.1, "avg_output_tokens": 1472.7, "avg_reasoning_tokens": 1329.5, "avg_cache_read_tokens": 626.5, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 30.3, "median_query_seconds": 14.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 86.09, "latency": 15.183, "stderr": 1.248, "cost_per_test": 0.015346, "avg_input_tokens": 1048.1, "avg_output_tokens": 1846.8, "avg_reasoning_tokens": 881.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 15.2, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 86.09, "latency": 23.815, "stderr": 1.225, "cost_per_test": 0.00459, "avg_input_tokens": 1057.0, "avg_output_tokens": 1443.6, "avg_cache_read_tokens": 750.7, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 23.8, "median_query_seconds": 17.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 86.09, "latency": 28.102, "stderr": 1.769, "cost_per_test": 0.000782, "avg_input_tokens": 1306.8, "avg_output_tokens": 778.8, "avg_reasoning_tokens": 581.8, "avg_cache_read_tokens": 1076.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 28.1, "median_query_seconds": 21.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 86.09, "latency": 32.444, "stderr": 1.225, "cost_per_test": 0.008575, "avg_input_tokens": 1065.1, "avg_output_tokens": 1745.6, "avg_reasoning_tokens": 1599.5, "avg_cache_read_tokens": 523.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 32.4, "median_query_seconds": 8.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 85.714, "latency": 4.655, "stderr": 1.239, "cost_per_test": 0.000934, "avg_input_tokens": 1080.8, "avg_output_tokens": 1003.0, "avg_reasoning_tokens": 759.2, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 4.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 85.714, "latency": 7.25, "stderr": 1.239, "cost_per_test": 0.008135, "avg_input_tokens": 1145.9, "avg_output_tokens": 313.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 7.3, "median_query_seconds": 7.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 85.714, "latency": 19.037, "stderr": 1.239, "cost_per_test": 0.000661, "avg_input_tokens": 1079.1, "avg_output_tokens": 1383.3, "avg_reasoning_tokens": 1198.5, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 19.0, "median_query_seconds": 4.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 85.589, "latency": 3.199, "stderr": 1.243, "cost_per_test": 0.002565, "avg_input_tokens": 1079.1, "avg_output_tokens": 896.6, "avg_reasoning_tokens": 760.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 3.2, "median_query_seconds": 2.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 85.589, "latency": 8.214, "stderr": 1.243, "cost_per_test": 0.001056, "avg_input_tokens": 1124.1, "avg_output_tokens": 695.1, "avg_reasoning_tokens": 647.7, "avg_cache_read_tokens": 836.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 8.2, "median_query_seconds": 3.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 85.589, "latency": 12.005, "stderr": 1.243, "cost_per_test": 0.00056, "avg_input_tokens": 1174.9, "avg_output_tokens": 931.7, "avg_reasoning_tokens": 829.8, "avg_cache_read_tokens": 940.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 12.0, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 85.589, "latency": 50.749, "stderr": 1.243, "cost_per_test": 0.006019, "avg_input_tokens": 1057.8, "avg_output_tokens": 1770.0, "avg_cache_read_tokens": 849.8, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 50.7, "median_query_seconds": 37.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 85.338, "latency": 7.022, "stderr": 1.252, "cost_per_test": 0.007685, "avg_input_tokens": 1153.4, "avg_output_tokens": 1172.7, "avg_reasoning_tokens": 1032.1, "avg_cache_read_tokens": 921.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 7.0, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 85.338, "latency": 22.345, "stderr": 1.252, "cost_per_test": 0.02236, "avg_input_tokens": 1173.9, "avg_output_tokens": 1255.9, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 22.3, "median_query_seconds": 17.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 85.088, "latency": 13.171, "stderr": 1.261, "cost_per_test": 0.007016, "avg_input_tokens": 1080.8, "avg_output_tokens": 566.5, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 13.2, "median_query_seconds": 12.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 84.837, "latency": 10.477, "stderr": 1.315, "cost_per_test": 0.001399, "avg_input_tokens": 1046.2, "avg_output_tokens": 991.7, "avg_reasoning_tokens": 935.2, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 10.5, "median_query_seconds": 5.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 84.837, "latency": 25.18, "stderr": 1.27, "cost_per_test": 0.003869, "avg_input_tokens": 1063.8, "avg_output_tokens": 432.1, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 25.2, "median_query_seconds": 20.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-0709": { "accuracy": 84.837, "latency": 39.89, "stderr": 1.27, "cost_per_test": 0.022401, "avg_input_tokens": 1025.9, "avg_output_tokens": 1288.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 39.9, "median_query_seconds": 19.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 84.837, "latency": 85.23, "stderr": 1.27, "cost_per_test": 0.004891, "avg_input_tokens": 1121.2, "avg_output_tokens": 1443.5, "avg_reasoning_tokens": 1350.8, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 85.2, "median_query_seconds": 31.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max": { "accuracy": 84.812, "latency": 0.0, "stderr": 1.251, "cost_per_test": 0.00375, "avg_input_tokens": 1063.9, "avg_output_tokens": 412.3, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 84.586, "latency": 8.534, "stderr": 1.278, "cost_per_test": 0.00095, "avg_input_tokens": 1080.8, "avg_output_tokens": 1019.5, "avg_reasoning_tokens": 769.2, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 8.5, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 84.586, "latency": 13.753, "stderr": 1.278, "cost_per_test": null, "avg_input_tokens": 1079.7, "avg_output_tokens": 1131.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 13.8, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 84.586, "latency": 41.38, "stderr": 1.404, "cost_per_test": 0.000171, "avg_input_tokens": 1045.1, "avg_output_tokens": 277.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 943.3, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 41.4, "median_query_seconds": 32.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 84.461, "latency": 58.865, "stderr": 1.282, "cost_per_test": 0.000616, "avg_input_tokens": 1045.1, "avg_output_tokens": 1305.1, "avg_reasoning_tokens": 1231.4, "avg_cache_read_tokens": 890.1, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 58.9, "median_query_seconds": 44.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 84.211, "latency": 13.894, "stderr": 1.299, "cost_per_test": 0.002023, "avg_input_tokens": 1046.2, "avg_output_tokens": 880.6, "avg_reasoning_tokens": 804.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 13.9, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 84.211, "latency": 17.866, "stderr": 1.291, "cost_per_test": 0.074227, "avg_input_tokens": 1046.2, "avg_output_tokens": 975.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 17.9, "median_query_seconds": 13.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 83.96, "latency": 36.831, "stderr": 1.299, "cost_per_test": 0.002473, "avg_input_tokens": 1054.6, "avg_output_tokens": 1983.7, "avg_cache_read_tokens": 933.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 772, "avg_turns": 1.0, "avg_query_seconds": 36.8, "median_query_seconds": 11.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 83.96, "latency": 76.522, "stderr": 1.299, "cost_per_test": 0.016928, "avg_input_tokens": 1065.8, "avg_output_tokens": 2608.2, "avg_reasoning_tokens": 2150.4, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 76.5, "median_query_seconds": 57.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-4.7": { "accuracy": 83.584, "latency": 64.321, "stderr": 1.315, "cost_per_test": 0.003926, "avg_input_tokens": 1058.1, "avg_output_tokens": 1640.6, "avg_cache_read_tokens": 649.8, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 64.3, "median_query_seconds": 37.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 83.333, "latency": 2.093, "stderr": 1.319, "cost_per_test": 0.003408, "avg_input_tokens": 1047.2, "avg_output_tokens": 164.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 2.1, "median_query_seconds": 1.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 83.333, "latency": 3.847, "stderr": 1.319, "cost_per_test": 0.001148, "avg_input_tokens": 1135.9, "avg_output_tokens": 1842.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 3.8, "median_query_seconds": 2.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 83.333, "latency": 25.992, "stderr": 1.323, "cost_per_test": 0.002098, "avg_input_tokens": 1198.0, "avg_output_tokens": 753.5, "avg_cache_read_tokens": 894.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 26.0, "median_query_seconds": 18.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 83.208, "latency": 5.9, "stderr": 1.323, "cost_per_test": 0.007899, "avg_input_tokens": 1145.9, "avg_output_tokens": 297.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 5.9, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 83.208, "latency": 24.811, "stderr": 1.323, "cost_per_test": 0.001243, "avg_input_tokens": 1079.2, "avg_output_tokens": 2837.8, "avg_reasoning_tokens": 2561.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 796, "avg_turns": 1.0, "avg_query_seconds": 24.8, "median_query_seconds": 20.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-4.6": { "accuracy": 83.208, "latency": 39.645, "stderr": 1.327, "cost_per_test": 0.004307, "avg_input_tokens": 1059.8, "avg_output_tokens": 1671.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 39.6, "median_query_seconds": 26.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 83.208, "latency": 64.574, "stderr": 1.35, "cost_per_test": 0.001324, "avg_input_tokens": 1056.8, "avg_output_tokens": 1611.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 64.6, "median_query_seconds": 43.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 82.957, "latency": 6.706, "stderr": 1.35, "cost_per_test": 0.005454, "avg_input_tokens": 1047.2, "avg_output_tokens": 283.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 6.7, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 82.957, "latency": 52.096, "stderr": 1.331, "cost_per_test": 0.022867, "avg_input_tokens": 1078.8, "avg_output_tokens": 2833.2, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 796, "avg_turns": 1.0, "avg_query_seconds": 52.1, "median_query_seconds": 47.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 82.832, "latency": 5.165, "stderr": 1.335, "cost_per_test": 0.00018, "avg_input_tokens": 1091.3, "avg_output_tokens": 669.0, "avg_cache_read_tokens": 809.5, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 5.2, "median_query_seconds": 3.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 82.707, "latency": 16.482, "stderr": 1.603, "cost_per_test": 0.007569, "avg_input_tokens": 1044.1, "avg_output_tokens": 1140.6, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 16.5, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 82.456, "latency": 11.527, "stderr": 1.346, "cost_per_test": 0.001865, "avg_input_tokens": 1066.0, "avg_output_tokens": 266.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 11.5, "median_query_seconds": 9.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 82.456, "latency": 14.49, "stderr": 1.346, "cost_per_test": 0.003107, "avg_input_tokens": 1046.2, "avg_output_tokens": 1422.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 14.5, "median_query_seconds": 12.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 82.206, "latency": 42.376, "stderr": 1.369, "cost_per_test": 0.001254, "avg_input_tokens": 1063.8, "avg_output_tokens": 1158.8, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 42.4, "median_query_seconds": 19.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 82.08, "latency": 18.433, "stderr": 1.358, "cost_per_test": 0.021302, "avg_input_tokens": 1173.9, "avg_output_tokens": 1185.3, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 18.4, "median_query_seconds": 15.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 81.83, "latency": 7.088, "stderr": 1.365, "cost_per_test": 0.004056, "avg_input_tokens": 1046.2, "avg_output_tokens": 660.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 7.1, "median_query_seconds": 6.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-2-1212": { "accuracy": 81.704, "latency": 3.194, "stderr": 1.369, "cost_per_test": 0.003577, "avg_input_tokens": 1046.8, "avg_output_tokens": 148.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 3.2, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-4.5": { "accuracy": 81.579, "latency": 70.368, "stderr": 1.372, "cost_per_test": 0.005607, "avg_input_tokens": 1058.1, "avg_output_tokens": 2259.9, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 70.4, "median_query_seconds": 61.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 81.454, "latency": 10.137, "stderr": 1.376, "cost_per_test": 0.001074, "avg_input_tokens": 1044.1, "avg_output_tokens": 148.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 10.1, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 81.328, "latency": 6.511, "stderr": 1.379, "cost_per_test": 0.00733, "avg_input_tokens": 1145.9, "avg_output_tokens": 259.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 6.5, "median_query_seconds": 6.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 81.078, "latency": 4.688, "stderr": 1.387, "cost_per_test": 0.000597, "avg_input_tokens": 1047.9, "avg_output_tokens": 415.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 81.078, "latency": 5.093, "stderr": 1.387, "cost_per_test": 0.000756, "avg_input_tokens": 1025.9, "avg_output_tokens": 896.3, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 5.1, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 81.078, "latency": 6.742, "stderr": 1.387, "cost_per_test": 0.000205, "avg_input_tokens": 1080.8, "avg_output_tokens": 1182.8, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 6.7, "median_query_seconds": 6.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 81.078, "latency": 7.35, "stderr": 1.387, "cost_per_test": 0.000978, "avg_input_tokens": 1025.9, "avg_output_tokens": 1340.5, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 7.3, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 81.078, "latency": 15.633, "stderr": 1.387, "cost_per_test": 0.000348, "avg_input_tokens": 1302.8, "avg_output_tokens": 1113.2, "avg_reasoning_tokens": 999.9, "avg_cache_read_tokens": 1067.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 15.6, "median_query_seconds": 10.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 80.952, "latency": 17.485, "stderr": 1.397, "cost_per_test": 0.008589, "avg_input_tokens": 1066.7, "avg_output_tokens": 1291.1, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 17.5, "median_query_seconds": 11.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 80.827, "latency": 14.281, "stderr": 1.394, "cost_per_test": 0.008321, "avg_input_tokens": 1046.2, "avg_output_tokens": 1629.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 14.3, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 80.702, "latency": 6.961, "stderr": 1.4, "cost_per_test": 0.005488, "avg_input_tokens": 1047.2, "avg_output_tokens": 287.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 7.0, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 80.576, "latency": 1.803, "stderr": 1.4, "cost_per_test": 0.001761, "avg_input_tokens": 1080.8, "avg_output_tokens": 82.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 1.8, "median_query_seconds": 1.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 80.576, "latency": 1.826, "stderr": 1.404, "cost_per_test": 0.000194, "avg_input_tokens": 1080.8, "avg_output_tokens": 214.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 1.8, "median_query_seconds": 1.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 80.576, "latency": 10.746, "stderr": 1.754, "cost_per_test": 0.001079, "avg_input_tokens": 1066.7, "avg_output_tokens": 363.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 10.7, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 80.451, "latency": 1.854, "stderr": 1.404, "cost_per_test": 0.000151, "avg_input_tokens": 1186.9, "avg_output_tokens": 103.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 919.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 1.9, "median_query_seconds": 1.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 80.326, "latency": 4.791, "stderr": 1.459, "cost_per_test": 0.000878, "avg_input_tokens": 1066.7, "avg_output_tokens": 225.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 4.8, "median_query_seconds": 4.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 79.825, "latency": 2.191, "stderr": 1.421, "cost_per_test": 0.000338, "avg_input_tokens": 1080.8, "avg_output_tokens": 574.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 2.2, "median_query_seconds": 2.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-3": { "accuracy": 79.825, "latency": 11.534, "stderr": 1.421, "cost_per_test": 0.012964, "avg_input_tokens": 1024.9, "avg_output_tokens": 659.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 11.5, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 79.825, "latency": 14.232, "stderr": 1.76, "cost_per_test": 0.008931, "avg_input_tokens": 1173.9, "avg_output_tokens": 1551.5, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 14.2, "median_query_seconds": 10.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 79.073, "latency": 25.216, "stderr": 1.44, "cost_per_test": 0.002921, "avg_input_tokens": 1063.0, "avg_output_tokens": 1056.3, "avg_cache_read_tokens": 36.0, "avg_cache_write_tokens": 1027.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 25.2, "median_query_seconds": 19.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 78.822, "latency": 2.333, "stderr": 1.446, "cost_per_test": 0.000888, "avg_input_tokens": 1047.2, "avg_output_tokens": 293.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 2.3, "median_query_seconds": 2.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 78.822, "latency": 3.148, "stderr": 1.446, "cost_per_test": 0.001019, "avg_input_tokens": 1044.1, "avg_output_tokens": 88.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 3.1, "median_query_seconds": 2.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 77.945, "latency": 23.345, "stderr": 1.485, "cost_per_test": 0.001847, "avg_input_tokens": 1043.0, "avg_output_tokens": 1218.2, "avg_cache_read_tokens": 16.0, "avg_cache_write_tokens": 1027.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 23.3, "median_query_seconds": 14.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 77.82, "latency": 1.579, "stderr": 1.471, "cost_per_test": 0.000896, "avg_input_tokens": 1145.9, "avg_output_tokens": 1332.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 1.6, "median_query_seconds": 1.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 77.82, "latency": 16.134, "stderr": 1.471, "cost_per_test": 0.001158, "avg_input_tokens": 1046.2, "avg_output_tokens": 2763.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 16.1, "median_query_seconds": 14.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 77.82, "latency": 52.376, "stderr": 1.477, "cost_per_test": 0.001118, "avg_input_tokens": 1114.2, "avg_output_tokens": 1587.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 52.4, "median_query_seconds": 46.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 77.193, "latency": 4.051, "stderr": 1.485, "cost_per_test": 0.003469, "avg_input_tokens": 1185.9, "avg_output_tokens": 182.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 4.1, "median_query_seconds": 3.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 77.068, "latency": 20.936, "stderr": 1.494, "cost_per_test": null, "avg_input_tokens": 1081.1, "avg_output_tokens": 838.5, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 20.9, "median_query_seconds": 18.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 76.316, "latency": 62.013, "stderr": 1.524, "cost_per_test": 0.000713, "avg_input_tokens": 1086.1, "avg_output_tokens": 1466.6, "avg_reasoning_tokens": 1260.1, "avg_cache_read_tokens": 910.7, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 62.0, "median_query_seconds": 39.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 75.815, "latency": 4.183, "stderr": 1.516, "cost_per_test": 0.001251, "avg_input_tokens": 1088.1, "avg_output_tokens": 333.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 4.2, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 75.564, "latency": 4.182, "stderr": 1.521, "cost_per_test": 0.000414, "avg_input_tokens": 1047.9, "avg_output_tokens": 382.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 4.2, "median_query_seconds": 4.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 75.564, "latency": 26.139, "stderr": 1.719, "cost_per_test": 0.00027, "avg_input_tokens": 1123.1, "avg_output_tokens": 1024.2, "avg_reasoning_tokens": 827.3, "avg_cache_read_tokens": 938.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 26.1, "median_query_seconds": 21.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 74.812, "latency": 3.55, "stderr": 1.629, "cost_per_test": 0.000169, "avg_input_tokens": 1046.2, "avg_output_tokens": 292.9, "avg_reasoning_tokens": 206.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 3.5, "median_query_seconds": 3.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 74.687, "latency": 1.181, "stderr": 1.539, "cost_per_test": 0.000126, "avg_input_tokens": 1080.8, "avg_output_tokens": 150.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 1.2, "median_query_seconds": 1.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 74.687, "latency": 7.087, "stderr": 1.612, "cost_per_test": 0.005887, "avg_input_tokens": 1082.1, "avg_output_tokens": 318.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 7.1, "median_query_seconds": 5.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 73.935, "latency": 18.032, "stderr": 1.681, "cost_per_test": null, "avg_input_tokens": 1081.1, "avg_output_tokens": 750.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 18.0, "median_query_seconds": 17.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 73.183, "latency": 2.151, "stderr": 1.568, "cost_per_test": 0.000141, "avg_input_tokens": 1066.7, "avg_output_tokens": 203.9, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 2.2, "median_query_seconds": 1.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 72.932, "latency": 2.934, "stderr": 1.573, "cost_per_test": 0.000277, "avg_input_tokens": 1047.2, "avg_output_tokens": 199.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 2.9, "median_query_seconds": 2.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 72.431, "latency": 5.807, "stderr": 1.582, "cost_per_test": 0.001922, "avg_input_tokens": 1145.9, "avg_output_tokens": 251.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 5.8, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 71.679, "latency": 1.642, "stderr": 1.603, "cost_per_test": 0.000228, "avg_input_tokens": 1047.2, "avg_output_tokens": 309.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 1.6, "median_query_seconds": 1.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 71.554, "latency": 42.466, "stderr": 1.661, "cost_per_test": 0.000615, "avg_input_tokens": 1114.2, "avg_output_tokens": 1789.6, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 42.5, "median_query_seconds": 35.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 71.429, "latency": 8.153, "stderr": 1.599, "cost_per_test": 0.001986, "avg_input_tokens": 1066.7, "avg_output_tokens": 968.5, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 8.2, "median_query_seconds": 5.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 68.546, "latency": 2.667, "stderr": 1.644, "cost_per_test": 0.000347, "avg_input_tokens": 1066.7, "avg_output_tokens": 223.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 2.7, "median_query_seconds": 2.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 64.912, "latency": 5.11, "stderr": 1.689, "cost_per_test": 0.00298, "avg_input_tokens": 1140.2, "avg_output_tokens": 87.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 5.1, "median_query_seconds": 3.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "cohere/command-r-plus": { "accuracy": 60.025, "latency": 3.13, "stderr": 1.734, "cost_per_test": 0.004253, "avg_input_tokens": 1082.1, "avg_output_tokens": 154.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 3.1, "median_query_seconds": 1.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 26.942, "latency": 0.978, "stderr": 1.571, "cost_per_test": 0.000259, "avg_input_tokens": 1140.2, "avg_output_tokens": 78.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 798, "avg_turns": 1.0, "avg_query_seconds": 1.0, "median_query_seconds": 0.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } } } }