{ "metadata": { "benchmark": "CorpFin v2", "slug": "corp_fin_v2", "description": "A private benchmark evaluating understanding of long-context credit agreements", "benchmark_id": "corp_fin_v2", "family": "corp_fin", "version": "2", "updated": "2026-08-12", "dataset_type": "private", "industry": "finance", "tasks": { "overall": "Overall", "exact_pages": "Exact Pages", "max_fitting_context": "Max Fitting Context", "shared_max_context": "Shared Max Context" }, "models": [ "ai21labs/jamba-1.5-large", "ai21labs/jamba-1.5-mini", "ai21labs/jamba-large-1.6", "ai21labs/jamba-mini-1.6", "alibaba/qwen3-max", "alibaba/qwen3-max-2026-01-23", "alibaba/qwen3-max-preview", "alibaba/qwen3.5-flash", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-27b", "alibaba/qwen3.6-max-preview", "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.8-max", "ant/ling-3.0-flash-2607", "anthropic/claude-3-5-haiku-20241022", "anthropic/claude-3-5-sonnet-20241022", "anthropic/claude-3-7-sonnet-20250219-thinking", "anthropic/claude-fable-5", "anthropic/claude-haiku-4-5-20251001", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "arcee-ai/trinity-large-thinking", "cohere/command-a-03-2025", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "fireworks/deepseek-r1", "fireworks/deepseek-v3", "fireworks/deepseek-v3-0324", "fireworks/deepseek-v3p1", "fireworks/deepseek-v3p2", "fireworks/deepseek-v3p2-thinking", "fireworks/gpt-oss-120b", "fireworks/gpt-oss-20b", "fireworks/llama4-maverick-instruct-basic", "fireworks/nemotron-lightning-3p5-30b-a3b", "google/gemini-1.5-flash-001", "google/gemini-1.5-flash-002", "google/gemini-1.5-pro-002", "google/gemini-2.0-flash-001", "google/gemini-2.0-pro-exp-02-05", "google/gemini-2.5-flash-lite-preview-09-2025", "google/gemini-2.5-flash-lite-preview-09-2025-thinking", "google/gemini-2.5-flash-preview-04-17", "google/gemini-2.5-flash-preview-09-2025", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-pro", "google/gemini-2.5-pro-exp-03-25", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "grok/grok-2-1212", "grok/grok-3", "grok/grok-3-mini-fast-high-reasoning", "grok/grok-3-mini-fast-low-reasoning", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "kimi/kimi-k2-thinking", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse_spark", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "minimax/MiniMax-M2.1", "minimax/MiniMax-M2.5", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/magistral-medium-2509", "mistralai/magistral-small-2509", "mistralai/mistral-large-2512", "mistralai/mistral-medium-2505", "mistralai/mistral-medium-3.5", "mistralai/mistral-small-2503", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-4.1-2025-04-14", "openai/gpt-4.1-mini-2025-04-14", "openai/gpt-4.1-nano-2025-04-14", "openai/gpt-4o-2024-08-06", "openai/gpt-4o-2024-11-20", "openai/gpt-4o-mini-2024-07-18", "openai/gpt-5-2025-08-07", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/o3-2025-04-16", "openai/o3-mini-2025-01-31", "openai/o4-mini-2025-04-16", "poolside/laguna-m.1", "poolside/laguna-xs.2", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "together/meta-llama/Llama-4-Scout-17B-16E-Instruct", "together/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", "together/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", "together/moonshotai/Kimi-K2-Instruct", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "zai/glm-4.5", "zai/glm-4.6", "zai/glm-4.7", "zai/glm-5-thinking", "zai/glm-5.1", "zai/glm-5.2" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "custom", "mode": "one-shot", "archived": false, "partner": false, "total_models": 134 }, "tasks": { "overall": { "anthropic/claude-opus-5": { "accuracy": 73.194, "latency": 28.573, "stderr": 0.871, "cost_per_test": 0.860452, "avg_input_tokens": 131326.9, "avg_output_tokens": 2067.5, "avg_cache_write_tokens": 129231.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 28.6, "median_query_seconds": 19.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 71.834, "latency": 52.99, "stderr": 0.883, "cost_per_test": 1.737573, "avg_input_tokens": 131326.9, "avg_output_tokens": 1935.6, "avg_cache_write_tokens": 131010.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 53.0, "median_query_seconds": 30.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 71.562, "latency": 37.99, "stderr": 0.888, "cost_per_test": 0.063819, "avg_input_tokens": 82805.2, "avg_output_tokens": 1108.0, "avg_reasoning_tokens": 693.1, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 38.0, "median_query_seconds": 30.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 71.29, "latency": 27.823, "stderr": 0.889, "cost_per_test": 0.10923, "avg_input_tokens": 81235.2, "avg_output_tokens": 1880.5, "avg_reasoning_tokens": 1474.3, "avg_cache_read_tokens": 278.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 27.8, "median_query_seconds": 24.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 70.94, "latency": 30.047, "stderr": 0.892, "cost_per_test": 0.108223, "avg_input_tokens": 81078.2, "avg_output_tokens": 1618.9, "avg_reasoning_tokens": 1126.8, "avg_cache_read_tokens": 4.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 30.1, "median_query_seconds": 16.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 69.619, "latency": 45.316, "stderr": 0.906, "cost_per_test": 0.025455, "avg_input_tokens": 78468.2, "avg_output_tokens": 1699.3, "avg_cache_read_tokens": 518.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 45.3, "median_query_seconds": 23.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 68.57, "latency": 29.199, "stderr": 0.915, "cost_per_test": 0.084258, "avg_input_tokens": 77962.7, "avg_output_tokens": 1626.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 29.2, "median_query_seconds": 15.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.3": { "accuracy": 68.532, "latency": 36.444, "stderr": 0.915, "cost_per_test": 0.134161, "avg_input_tokens": 78004.5, "avg_output_tokens": 1630.8, "avg_reasoning_tokens": 1240.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 36.4, "median_query_seconds": 37.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 68.415, "latency": 23.304, "stderr": 0.916, "cost_per_test": 0.43166, "avg_input_tokens": 79303.7, "avg_output_tokens": 1171.9, "avg_reasoning_tokens": 909.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 23.3, "median_query_seconds": 14.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 68.259, "latency": 80.779, "stderr": 0.917, "cost_per_test": 0.052412, "avg_input_tokens": 80662.7, "avg_output_tokens": 1456.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 80.8, "median_query_seconds": 66.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 68.104, "latency": 24.459, "stderr": 0.918, "cost_per_test": 0.049736, "avg_input_tokens": 79564.8, "avg_output_tokens": 855.1, "avg_cache_read_tokens": 113.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 24.5, "median_query_seconds": 18.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 68.026, "latency": 121.317, "stderr": 0.918, "cost_per_test": 0.261088, "avg_input_tokens": 80812.6, "avg_output_tokens": 3142.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 121.3, "median_query_seconds": 114.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 67.405, "latency": 91.881, "stderr": 0.924, "cost_per_test": 0.221291, "avg_input_tokens": 79581.3, "avg_output_tokens": 1038.2, "avg_reasoning_tokens": 777.6, "avg_cache_read_tokens": 208.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 91.9, "median_query_seconds": 91.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 67.016, "latency": 20.581, "stderr": 0.926, "cost_per_test": 0.487461, "avg_input_tokens": 74402.8, "avg_output_tokens": 922.9, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 20.6, "median_query_seconds": 16.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 66.977, "latency": 18.628, "stderr": 0.926, "cost_per_test": 0.516119, "avg_input_tokens": 130282.8, "avg_output_tokens": 1842.2, "avg_cache_write_tokens": 130246.5, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 18.6, "median_query_seconds": 12.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 66.9, "latency": 11.839, "stderr": 0.925, "cost_per_test": 0.040289, "avg_input_tokens": 56833.1, "avg_output_tokens": 57845.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 11.8, "median_query_seconds": 9.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 66.744, "latency": 79.59, "stderr": 0.928, "cost_per_test": 0.07328, "avg_input_tokens": 79472.2, "avg_output_tokens": 1732.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 11582.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 79.6, "median_query_seconds": 71.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 48000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 66.706, "latency": 20.226, "stderr": 0.928, "cost_per_test": 0.855534, "avg_input_tokens": 131326.9, "avg_output_tokens": 1449.7, "avg_cache_read_tokens": 192.5, "avg_cache_write_tokens": 130818.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.2, "median_query_seconds": 16.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-max-preview": { "accuracy": 66.473, "latency": 47.068, "stderr": 0.929, "cost_per_test": 0.156029, "avg_input_tokens": 78674.0, "avg_output_tokens": 1700.4, "avg_reasoning_tokens": 1395.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 47.1, "median_query_seconds": 41.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 66.434, "latency": 11.279, "stderr": 0.93, "cost_per_test": 0.047154, "avg_input_tokens": 85013.4, "avg_output_tokens": 1549.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 11.3, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 66.161, "latency": 24.256, "stderr": 0.932, "cost_per_test": 0.165532, "avg_input_tokens": 79582.8, "avg_output_tokens": 1187.3, "avg_reasoning_tokens": 965.3, "avg_cache_read_tokens": 504.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 24.3, "median_query_seconds": 19.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.2": { "accuracy": 66.123, "latency": 24.586, "stderr": 0.932, "cost_per_test": 0.121857, "avg_input_tokens": 82885.5, "avg_output_tokens": 1322.5, "avg_reasoning_tokens": 1030.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 24.6, "median_query_seconds": 18.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 66.084, "latency": 17.75, "stderr": 0.933, "cost_per_test": 0.816017, "avg_input_tokens": 129932.5, "avg_output_tokens": 1059.6, "avg_cache_read_tokens": 1416.6, "avg_cache_write_tokens": 128195.4, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 17.7, "median_query_seconds": 14.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4-0709": { "accuracy": 66.045, "latency": 29.733, "stderr": 0.932, "cost_per_test": 0.190665, "avg_input_tokens": 56723.2, "avg_output_tokens": 1366.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 29.7, "median_query_seconds": 22.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 65.967, "latency": 28.411, "stderr": 0.934, "cost_per_test": 0.015643, "avg_input_tokens": 76969.7, "avg_output_tokens": 523.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 28.4, "median_query_seconds": 20.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 65.889, "latency": 26.038, "stderr": 0.934, "cost_per_test": 0.153756, "avg_input_tokens": 80057.0, "avg_output_tokens": 1739.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 26.0, "median_query_seconds": 16.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 65.851, "latency": 46.649, "stderr": 0.935, "cost_per_test": 0.179323, "avg_input_tokens": 84284.7, "avg_output_tokens": 1792.2, "avg_reasoning_tokens": 1437.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 46.7, "median_query_seconds": 28.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 65.462, "latency": 15.015, "stderr": 0.937, "cost_per_test": null, "avg_input_tokens": 90638.9, "avg_output_tokens": 1114.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 15.0, "median_query_seconds": 12.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 65.423, "latency": 32.854, "stderr": 0.937, "cost_per_test": 0.002183, "avg_input_tokens": 81375.8, "avg_output_tokens": 2138.5, "avg_reasoning_tokens": 1927.6, "avg_cache_read_tokens": 81311.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 32.8, "median_query_seconds": 18.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 65.307, "latency": 7.717, "stderr": 0.938, "cost_per_test": 0.171459, "avg_input_tokens": 81695.7, "avg_output_tokens": 686.5, "avg_reasoning_tokens": 439.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 7.7, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 65.307, "latency": 18.483, "stderr": 0.938, "cost_per_test": 0.307405, "avg_input_tokens": 78408.7, "avg_output_tokens": 972.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.5, "median_query_seconds": 13.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 65.307, "latency": 155.861, "stderr": 0.936, "cost_per_test": 0.058073, "avg_input_tokens": 83902.4, "avg_output_tokens": 3167.6, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 155.9, "median_query_seconds": 42.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 65.268, "latency": 82.34, "stderr": 0.938, "cost_per_test": 0.235266, "avg_input_tokens": 81696.4, "avg_output_tokens": 2068.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 82.4, "median_query_seconds": 41.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark": { "accuracy": 65.113, "latency": 40.057, "stderr": 0.936, "cost_per_test": 0.011749, "avg_input_tokens": 70089.9, "avg_output_tokens": 2060.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 40.1, "median_query_seconds": 33.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 65.074, "latency": 17.725, "stderr": 0.939, "cost_per_test": 0.173423, "avg_input_tokens": 24611.5, "avg_output_tokens": 809.6, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 17.7, "median_query_seconds": 15.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 64.686, "latency": 8.7, "stderr": 0.942, "cost_per_test": 0.140208, "avg_input_tokens": 85013.4, "avg_output_tokens": 1409.8, "avg_reasoning_tokens": 1145.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 8.7, "median_query_seconds": 6.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 64.491, "latency": 25.388, "stderr": 0.942, "cost_per_test": 0.181957, "avg_input_tokens": 85013.4, "avg_output_tokens": 994.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 25.4, "median_query_seconds": 18.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 64.452, "latency": 68.17, "stderr": 0.943, "cost_per_test": 0.078922, "avg_input_tokens": 73244.6, "avg_output_tokens": 1884.4, "avg_reasoning_tokens": 1594.3, "avg_cache_read_tokens": 440.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 68.2, "median_query_seconds": 29.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 64.375, "latency": 15.941, "stderr": 0.943, "cost_per_test": 0.484982, "avg_input_tokens": 81699.2, "avg_output_tokens": 1246.0, "avg_reasoning_tokens": 1025.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 15.9, "median_query_seconds": 7.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 64.219, "latency": 16.968, "stderr": 0.945, "cost_per_test": 0.019039, "avg_input_tokens": 81695.7, "avg_output_tokens": 2251.5, "avg_reasoning_tokens": 2007.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 17.0, "median_query_seconds": 8.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 63.831, "latency": 35.289, "stderr": 0.947, "cost_per_test": 0.091991, "avg_input_tokens": 69046.2, "avg_output_tokens": 692.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 35.3, "median_query_seconds": 27.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 63.714, "latency": 19.934, "stderr": 0.946, "cost_per_test": 0.219883, "avg_input_tokens": 84246.7, "avg_output_tokens": 1235.5, "avg_reasoning_tokens": 972.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 19.9, "median_query_seconds": 15.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 63.675, "latency": 9.67, "stderr": 0.947, "cost_per_test": 0.163386, "avg_input_tokens": 78056.4, "avg_output_tokens": 1212.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 9.7, "median_query_seconds": 8.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 63.675, "latency": 20.059, "stderr": 0.946, "cost_per_test": 0.179069, "avg_input_tokens": 84937.7, "avg_output_tokens": 773.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.1, "median_query_seconds": 16.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 63.559, "latency": 35.198, "stderr": 0.947, "cost_per_test": 0.009638, "avg_input_tokens": 84247.7, "avg_output_tokens": 3033.8, "avg_reasoning_tokens": 2723.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 35.2, "median_query_seconds": 32.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 63.326, "latency": 6.329, "stderr": 0.949, "cost_per_test": 0.136842, "avg_input_tokens": 85013.4, "avg_output_tokens": 1243.0, "avg_reasoning_tokens": 989.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 6.3, "median_query_seconds": 5.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 63.054, "latency": 10.14, "stderr": 0.951, "cost_per_test": 0.13202, "avg_input_tokens": 63955.8, "avg_output_tokens": 513.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 10.2, "median_query_seconds": 7.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 62.898, "latency": 68.577, "stderr": 0.95, "cost_per_test": 0.067926, "avg_input_tokens": 60264.4, "avg_output_tokens": 2404.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 68.6, "median_query_seconds": 54.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 62.315, "latency": 62.759, "stderr": 0.954, "cost_per_test": 0.053549, "avg_input_tokens": 78675.4, "avg_output_tokens": 1762.1, "avg_reasoning_tokens": 1508.4, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 62.8, "median_query_seconds": 60.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 61.966, "latency": 20.544, "stderr": 0.956, "cost_per_test": 0.255071, "avg_input_tokens": 74438.4, "avg_output_tokens": 910.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.6, "median_query_seconds": 17.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 61.927, "latency": 12.451, "stderr": 0.957, "cost_per_test": 0.006376, "avg_input_tokens": 82349.7, "avg_output_tokens": 909.1, "avg_cache_read_tokens": 0.6, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 12.5, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 61.927, "latency": 35.839, "stderr": 0.957, "cost_per_test": 0.047203, "avg_input_tokens": 84246.7, "avg_output_tokens": 1693.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 35.8, "median_query_seconds": 30.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 61.849, "latency": 16.477, "stderr": 0.957, "cost_per_test": 0.011801, "avg_input_tokens": 81362.8, "avg_output_tokens": 1476.8, "avg_reasoning_tokens": 1277.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 16.5, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 61.422, "latency": 18.899, "stderr": 0.959, "cost_per_test": 0.037238, "avg_input_tokens": 84763.5, "avg_output_tokens": 664.2, "avg_reasoning_tokens": 354.9, "avg_cache_read_tokens": 491.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.9, "median_query_seconds": 16.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 61.383, "latency": 79.331, "stderr": 0.959, "cost_per_test": 0.148855, "avg_input_tokens": 81362.8, "avg_output_tokens": 2409.6, "avg_reasoning_tokens": 2206.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 79.3, "median_query_seconds": 37.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 61.305, "latency": 9.401, "stderr": 0.96, "cost_per_test": 0.150159, "avg_input_tokens": 24593.2, "avg_output_tokens": 242.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 9.4, "median_query_seconds": 8.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 61.228, "latency": 168.523, "stderr": 0.959, "cost_per_test": 0.236746, "avg_input_tokens": 74438.1, "avg_output_tokens": 895.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 168.5, "median_query_seconds": 60.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 61.189, "latency": 9.397, "stderr": 0.961, "cost_per_test": 0.01612, "avg_input_tokens": 79304.1, "avg_output_tokens": 526.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 9.4, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 61.189, "latency": 10.973, "stderr": 0.96, "cost_per_test": 0.022663, "avg_input_tokens": 58492.4, "avg_output_tokens": 614.2, "avg_cache_read_tokens": 26.4, "avg_cache_write_tokens": 58466.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 11.0, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 61.111, "latency": 14.386, "stderr": 0.957, "cost_per_test": 0.017292, "avg_input_tokens": 55110.9, "avg_output_tokens": 1518.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 14.4, "median_query_seconds": 12.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 61.072, "latency": 160.705, "stderr": 0.961, "cost_per_test": 0.086116, "avg_input_tokens": 67070.5, "avg_output_tokens": 227.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 160.7, "median_query_seconds": 143.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 61.033, "latency": 48.842, "stderr": 0.96, "cost_per_test": 0.024166, "avg_input_tokens": 47634.6, "avg_output_tokens": 424.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 48.8, "median_query_seconds": 36.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-4.5": { "accuracy": 60.956, "latency": 95.244, "stderr": 0.959, "cost_per_test": 0.03783, "avg_input_tokens": 59992.8, "avg_output_tokens": 938.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 95.2, "median_query_seconds": 76.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 60.917, "latency": 52.297, "stderr": 0.962, "cost_per_test": 0.076368, "avg_input_tokens": 79305.7, "avg_output_tokens": 3753.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 52.3, "median_query_seconds": 28.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 60.8, "latency": 13.383, "stderr": 0.962, "cost_per_test": 0.237245, "avg_input_tokens": 78268.8, "avg_output_tokens": 266.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 13.4, "median_query_seconds": 11.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 60.8, "latency": 27.425, "stderr": 0.961, "cost_per_test": 0.09535, "avg_input_tokens": 74365.7, "avg_output_tokens": 239.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 27.4, "median_query_seconds": 24.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 60.684, "latency": 3.836, "stderr": 0.962, "cost_per_test": 0.028548, "avg_input_tokens": 85013.4, "avg_output_tokens": 1217.5, "avg_reasoning_tokens": 997.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 3.8, "median_query_seconds": 3.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 60.606, "latency": 14.242, "stderr": 0.962, "cost_per_test": 0.089265, "avg_input_tokens": 78410.0, "avg_output_tokens": 985.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 14.2, "median_query_seconds": 12.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 60.567, "latency": 93.297, "stderr": 0.962, "cost_per_test": 0.038163, "avg_input_tokens": 59961.4, "avg_output_tokens": 1665.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 93.3, "median_query_seconds": 74.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 60.412, "latency": 22.163, "stderr": 0.964, "cost_per_test": 0.232197, "avg_input_tokens": 72645.0, "avg_output_tokens": 950.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 22.1, "median_query_seconds": 21.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 18384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-4-5-20251001": { "accuracy": 60.295, "latency": 7.951, "stderr": 0.964, "cost_per_test": 0.075779, "avg_input_tokens": 74410.2, "avg_output_tokens": 273.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 7.9, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 60.179, "latency": 73.788, "stderr": 0.965, "cost_per_test": 0.020976, "avg_input_tokens": 71269.1, "avg_output_tokens": 1583.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 73.8, "median_query_seconds": 43.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 59.907, "latency": 17.625, "stderr": 0.966, "cost_per_test": 0.011733, "avg_input_tokens": 84758.2, "avg_output_tokens": 598.6, "avg_reasoning_tokens": 292.3, "avg_cache_read_tokens": 2191.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 17.6, "median_query_seconds": 16.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 59.829, "latency": 16.302, "stderr": 0.966, "cost_per_test": 0.08579, "avg_input_tokens": 66572.7, "avg_output_tokens": 257.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 16.3, "median_query_seconds": 10.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 59.751, "latency": 51.757, "stderr": 0.966, "cost_per_test": 0.026119, "avg_input_tokens": 85043.1, "avg_output_tokens": 242.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 51.8, "median_query_seconds": 48.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 59.713, "latency": 19.153, "stderr": 0.965, "cost_per_test": 0.133487, "avg_input_tokens": 63954.8, "avg_output_tokens": 697.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 19.2, "median_query_seconds": 15.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3": { "accuracy": 59.713, "latency": 24.354, "stderr": 0.965, "cost_per_test": 0.169809, "avg_input_tokens": 55109.9, "avg_output_tokens": 298.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 24.4, "median_query_seconds": 23.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 59.596, "latency": 16.599, "stderr": 0.966, "cost_per_test": 0.024284, "avg_input_tokens": 58564.1, "avg_output_tokens": 972.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 16.6, "median_query_seconds": 13.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 59.479, "latency": 10.516, "stderr": 0.96, "cost_per_test": 0.01706, "avg_input_tokens": 55110.9, "avg_output_tokens": 1053.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 10.5, "median_query_seconds": 9.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 59.363, "latency": 4.03, "stderr": 0.967, "cost_per_test": 0.022927, "avg_input_tokens": 85013.4, "avg_output_tokens": 1147.7, "avg_reasoning_tokens": 934.1, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 4.0, "median_query_seconds": 3.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 58.974, "latency": 17.528, "stderr": 0.967, "cost_per_test": 0.074104, "avg_input_tokens": 63954.8, "avg_output_tokens": 853.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 17.5, "median_query_seconds": 14.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 58.974, "latency": 51.895, "stderr": 0.969, "cost_per_test": 0.026123, "avg_input_tokens": 85043.1, "avg_output_tokens": 244.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 51.9, "median_query_seconds": 49.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 58.896, "latency": 21.969, "stderr": 0.968, "cost_per_test": 0.022461, "avg_input_tokens": 56445.3, "avg_output_tokens": 1091.1, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 22.0, "median_query_seconds": 15.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 58.78, "latency": 43.699, "stderr": 0.969, "cost_per_test": 0.14574, "avg_input_tokens": 84672.1, "avg_output_tokens": 2497.6, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 43.7, "median_query_seconds": 40.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 58.392, "latency": 9.37, "stderr": 0.967, "cost_per_test": 0.039974, "avg_input_tokens": 56843.1, "avg_output_tokens": 57211.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 9.3, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 58.236, "latency": 42.722, "stderr": 0.968, "cost_per_test": 0.009601, "avg_input_tokens": 59386.1, "avg_output_tokens": 1173.2, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 42.7, "median_query_seconds": 41.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 58.159, "latency": 211.713, "stderr": 0.971, "cost_per_test": null, "avg_input_tokens": 84444.0, "avg_output_tokens": 1022.5, "avg_reasoning_tokens": 664.8, "avg_cache_read_tokens": 611.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 211.7, "median_query_seconds": 197.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 57.926, "latency": 6.443, "stderr": 0.972, "cost_per_test": 0.026113, "avg_input_tokens": 63955.8, "avg_output_tokens": 331.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 6.4, "median_query_seconds": 5.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 57.576, "latency": 54.225, "stderr": 0.973, "cost_per_test": 0.008596, "avg_input_tokens": 85043.1, "avg_output_tokens": 230.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 54.2, "median_query_seconds": 50.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-4.6": { "accuracy": 56.838, "latency": 78.061, "stderr": 0.97, "cost_per_test": 0.04461, "avg_input_tokens": 69974.4, "avg_output_tokens": 1432.1, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 78.1, "median_query_seconds": 45.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 56.333, "latency": 188.244, "stderr": 0.976, "cost_per_test": null, "avg_input_tokens": 84516.3, "avg_output_tokens": 909.2, "avg_reasoning_tokens": 548.6, "avg_cache_read_tokens": 43.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 188.3, "median_query_seconds": 181.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 56.294, "latency": 54.75, "stderr": 0.976, "cost_per_test": 0.008646, "avg_input_tokens": 85043.1, "avg_output_tokens": 354.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 54.7, "median_query_seconds": 52.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max": { "accuracy": 55.944, "latency": 55.675, "stderr": 0.977, "cost_per_test": 0.086893, "avg_input_tokens": 71433.1, "avg_output_tokens": 195.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 55.7, "median_query_seconds": 53.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 54.74, "latency": 36.732, "stderr": 0.978, "cost_per_test": 0.052049, "avg_input_tokens": 57572.1, "avg_output_tokens": 260.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 36.7, "median_query_seconds": 29.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 54.701, "latency": 186.345, "stderr": 0.98, "cost_per_test": 0.227257, "avg_input_tokens": 74410.1, "avg_output_tokens": 268.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 186.3, "median_query_seconds": 44.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "arcee-ai/trinity-large-thinking": { "accuracy": 54.662, "latency": 12.482, "stderr": 0.977, "cost_per_test": 0.01939, "avg_input_tokens": 75141.2, "avg_output_tokens": 819.5, "avg_reasoning_tokens": 365.1, "avg_cache_read_tokens": 128.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 12.5, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Arcee-Ai", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 54.235, "latency": 38.792, "stderr": 0.98, "cost_per_test": 0.004794, "avg_input_tokens": 84587.1, "avg_output_tokens": 3058.9, "avg_cache_read_tokens": 1164.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 38.8, "median_query_seconds": 21.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 54.157, "latency": 8.992, "stderr": 0.982, "cost_per_test": 0.020528, "avg_input_tokens": 66474.9, "avg_output_tokens": 234.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 9.0, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 54.118, "latency": 46.774, "stderr": 0.979, "cost_per_test": 0.086732, "avg_input_tokens": 61432.4, "avg_output_tokens": 714.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 46.8, "median_query_seconds": 44.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 53.613, "latency": 0.049, "stderr": 0.981, "cost_per_test": 0.23772, "avg_input_tokens": 78320.2, "avg_output_tokens": 184.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.0, "median_query_seconds": 0.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 53.147, "latency": 33.968, "stderr": 0.979, "cost_per_test": 0.004549, "avg_input_tokens": 54803.5, "avg_output_tokens": 2375.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 34.0, "median_query_seconds": 31.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 52.953, "latency": 227.451, "stderr": 0.975, "cost_per_test": 0.09654, "avg_input_tokens": 79511.4, "avg_output_tokens": 187.6, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 227.5, "median_query_seconds": 163.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 52.486, "latency": 13.465, "stderr": 0.971, "cost_per_test": 0.015705, "avg_input_tokens": 78036.1, "avg_output_tokens": 214.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 13.5, "median_query_seconds": 11.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 52.486, "latency": 28.322, "stderr": 0.98, "cost_per_test": 0.052026, "avg_input_tokens": 57572.1, "avg_output_tokens": 234.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 28.3, "median_query_seconds": 22.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-v3p1": { "accuracy": 51.476, "latency": 62.999, "stderr": 0.982, "cost_per_test": 0.037369, "avg_input_tokens": 66151.3, "avg_output_tokens": 193.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 63.0, "median_query_seconds": 58.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-2-1212": { "accuracy": 51.126, "latency": 87.134, "stderr": 0.981, "cost_per_test": 0.115097, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 50.971, "latency": 99.485, "stderr": 0.979, "cost_per_test": 0.038215, "avg_input_tokens": 65175.5, "avg_output_tokens": 1048.4, "turn_metrics": { "tasks_measured": null, "avg_turns": 1.0, "avg_query_seconds": 99.5, "median_query_seconds": 75.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 50.816, "latency": 0.039, "stderr": 0.981, "cost_per_test": 0.063315, "avg_input_tokens": 78320.2, "avg_output_tokens": 164.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.0, "median_query_seconds": 0.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 50.738, "latency": 30.346, "stderr": 0.974, "cost_per_test": 0.024082, "avg_input_tokens": 59013.1, "avg_output_tokens": 238.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 30.3, "median_query_seconds": 21.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 50.388, "latency": 40.894, "stderr": 0.985, "cost_per_test": 0.059336, "avg_input_tokens": 59011.2, "avg_output_tokens": 108.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 40.9, "median_query_seconds": 31.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 49.728, "latency": 4.724, "stderr": 0.981, "cost_per_test": 0.014178, "avg_input_tokens": 63418.5, "avg_output_tokens": 256.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 4.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 47.941, "latency": 133.787, "stderr": 0.98, "cost_per_test": 0.036826, "avg_input_tokens": 65362.4, "avg_output_tokens": 140.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 133.8, "median_query_seconds": 66.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 47.397, "latency": 55.265, "stderr": 0.982, "cost_per_test": 0.125244, "avg_input_tokens": 58977.6, "avg_output_tokens": 1872.7, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 55.3, "median_query_seconds": 43.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 46.776, "latency": 9.697, "stderr": 0.98, "cost_per_test": 0.011639, "avg_input_tokens": 63418.5, "avg_output_tokens": 380.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 9.7, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-4.7": { "accuracy": 46.387, "latency": 66.101, "stderr": 0.899, "cost_per_test": 0.039082, "avg_input_tokens": 62817.6, "avg_output_tokens": 2602.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 66.1, "median_query_seconds": 47.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 45.96, "latency": 14.146, "stderr": 0.972, "cost_per_test": 0.206138, "avg_input_tokens": 81596.0, "avg_output_tokens": 214.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 14.1, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 45.921, "latency": 6.461, "stderr": 0.973, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 45.455, "latency": 0.041, "stderr": 0.977, "cost_per_test": 0.008848, "avg_input_tokens": 58924.9, "avg_output_tokens": 126.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.1, "median_query_seconds": 0.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 45.299, "latency": 31.047, "stderr": 0.964, "cost_per_test": 0.078121, "avg_input_tokens": 67218.5, "avg_output_tokens": 950.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 31.0, "median_query_seconds": 28.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 44.173, "latency": 11.291, "stderr": 0.972, "cost_per_test": 0.004408, "avg_input_tokens": 57648.6, "avg_output_tokens": 280.3, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 11.3, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 44.017, "latency": 14.053, "stderr": 0.965, "cost_per_test": 0.030437, "avg_input_tokens": 59005.9, "avg_output_tokens": 646.5, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 14.1, "median_query_seconds": 11.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.0-pro-exp-02-05": { "accuracy": 43.435, "latency": 18.879, "stderr": 0.971, "cost_per_test": 0.104585, "avg_input_tokens": 83184.2, "avg_output_tokens": 120.9, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.9, "median_query_seconds": 14.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 42.075, "latency": 4.954, "stderr": 0.968, "cost_per_test": 0.006454, "avg_input_tokens": 63955.8, "avg_output_tokens": 146.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 4.9, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 41.531, "latency": 27.38, "stderr": 0.963, "cost_per_test": 0.150029, "avg_input_tokens": 74185.3, "avg_output_tokens": 207.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 27.4, "median_query_seconds": 11.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 40.521, "latency": 38.05, "stderr": 0.967, "cost_per_test": 0.131149, "avg_input_tokens": 104415.6, "avg_output_tokens": 125.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 38.1, "median_query_seconds": 30.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 39.433, "latency": 0.046, "stderr": 0.951, "cost_per_test": 0.147391, "avg_input_tokens": 58924.9, "avg_output_tokens": 117.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.1, "median_query_seconds": 0.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ai21labs/jamba-1.5-large": { "accuracy": 39.433, "latency": 10.765, "stderr": 0.959, "cost_per_test": 0.149067, "avg_input_tokens": 74185.3, "avg_output_tokens": 87.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 10.8, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "together/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { "accuracy": 38.85, "latency": 0.062, "stderr": 0.95, "cost_per_test": 0.053099, "avg_input_tokens": 60219.7, "avg_output_tokens": 119.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.1, "median_query_seconds": 0.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 38.19, "latency": 28.407, "stderr": 0.958, "cost_per_test": 0.007862, "avg_input_tokens": 104415.6, "avg_output_tokens": 103.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 28.4, "median_query_seconds": 18.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 38.034, "latency": 4.248, "stderr": 0.951, "cost_per_test": 0.014897, "avg_input_tokens": 74185.3, "avg_output_tokens": 150.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 4.3, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "together/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { "accuracy": 37.801, "latency": 0.038, "stderr": 0.934, "cost_per_test": 0.010777, "avg_input_tokens": 59649.1, "avg_output_tokens": 225.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.0, "median_query_seconds": 0.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "ai21labs/jamba-1.5-mini": { "accuracy": 33.877, "latency": 2.315, "stderr": 0.929, "cost_per_test": 0.014865, "avg_input_tokens": 74185.3, "avg_output_tokens": 68.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 2.3, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 33.722, "latency": 30.038, "stderr": 0.93, "cost_per_test": 0.010481, "avg_input_tokens": 104415.6, "avg_output_tokens": 97.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 30.0, "median_query_seconds": 14.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-1.5-flash-001": { "accuracy": 28.633, "latency": 1.156, "stderr": 0.88, "cost_per_test": 0.006413, "avg_input_tokens": 85159.8, "avg_output_tokens": 84.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 1.2, "median_query_seconds": 0.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "exact_pages": { "kimi/kimi-k3": { "accuracy": 69.347, "latency": 19.173, "stderr": 1.574, "cost_per_test": 0.012378, "avg_input_tokens": 1480.5, "avg_output_tokens": 769.0, "avg_reasoning_tokens": 438.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 19.2, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 68.881, "latency": 18.519, "stderr": 1.581, "cost_per_test": 0.006684, "avg_input_tokens": 1391.8, "avg_output_tokens": 1330.7, "avg_cache_read_tokens": 117.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.5, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 68.648, "latency": 16.028, "stderr": 1.584, "cost_per_test": 0.044928, "avg_input_tokens": 2224.4, "avg_output_tokens": 1241.6, "avg_reasoning_tokens": 408.8, "avg_cache_write_tokens": 2213.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 16.0, "median_query_seconds": 10.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 68.648, "latency": 38.507, "stderr": 1.584, "cost_per_test": 0.001897, "avg_input_tokens": 1391.8, "avg_output_tokens": 1258.2, "avg_cache_read_tokens": 128.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 38.5, "median_query_seconds": 19.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 68.065, "latency": 11.684, "stderr": 1.592, "cost_per_test": 0.098516, "avg_input_tokens": 187158.5, "avg_output_tokens": 1645.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 11.7, "median_query_seconds": 8.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 68.065, "latency": 33.05, "stderr": 1.592, "cost_per_test": 0.335055, "avg_input_tokens": 174905.2, "avg_output_tokens": 2069.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 33.0, "median_query_seconds": 21.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 67.249, "latency": 8.663, "stderr": 1.602, "cost_per_test": 0.001343, "avg_input_tokens": 1434.7, "avg_output_tokens": 2112.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 8.7, "median_query_seconds": 6.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 67.016, "latency": 5.678, "stderr": 1.605, "cost_per_test": 0.000896, "avg_input_tokens": 1306.6, "avg_output_tokens": 1008.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 5.7, "median_query_seconds": 5.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 66.9, "latency": 18.601, "stderr": 1.607, "cost_per_test": 0.090437, "avg_input_tokens": 2224.4, "avg_output_tokens": 1300.0, "avg_cache_write_tokens": 1277.9, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.6, "median_query_seconds": 12.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 66.783, "latency": 45.867, "stderr": 1.608, "cost_per_test": 0.004871, "avg_input_tokens": 1330.7, "avg_output_tokens": 1282.9, "avg_reasoning_tokens": 983.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 45.9, "median_query_seconds": 67.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 66.434, "latency": 18.683, "stderr": 1.612, "cost_per_test": 0.009465, "avg_input_tokens": 1527.2, "avg_output_tokens": 1819.5, "avg_reasoning_tokens": 1494.7, "avg_cache_read_tokens": 160.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.7, "median_query_seconds": 15.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.5": { "accuracy": 66.317, "latency": 154.026, "stderr": 1.614, "cost_per_test": 0.007543, "avg_input_tokens": 1377.8, "avg_output_tokens": 799.4, "avg_reasoning_tokens": 574.2, "avg_cache_read_tokens": 6.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 154.0, "median_query_seconds": 156.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 66.2, "latency": 40.528, "stderr": 1.615, "cost_per_test": 0.004139, "avg_input_tokens": 1402.5, "avg_output_tokens": 1099.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 40.5, "median_query_seconds": 30.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 66.084, "latency": 17.558, "stderr": 1.616, "cost_per_test": 0.006998, "avg_input_tokens": 1370.2, "avg_output_tokens": 1245.3, "avg_reasoning_tokens": 883.8, "avg_cache_read_tokens": 6.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 17.6, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4-0709": { "accuracy": 65.967, "latency": 11.944, "stderr": 1.618, "cost_per_test": 0.021639, "avg_input_tokens": 1324.8, "avg_output_tokens": 1177.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 11.9, "median_query_seconds": 3.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 65.851, "latency": 7.748, "stderr": 1.619, "cost_per_test": 0.001098, "avg_input_tokens": 1306.6, "avg_output_tokens": 1412.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 7.7, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 65.618, "latency": 17.821, "stderr": 1.622, "cost_per_test": 0.02963, "avg_input_tokens": 1384.1, "avg_output_tokens": 757.0, "avg_reasoning_tokens": 538.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 17.8, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 65.501, "latency": 32.826, "stderr": 1.623, "cost_per_test": 0.033844, "avg_input_tokens": 168465.0, "avg_output_tokens": 302.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 32.8, "median_query_seconds": 22.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 64.918, "latency": 10.405, "stderr": 1.629, "cost_per_test": 0.022285, "avg_input_tokens": 1593.2, "avg_output_tokens": 558.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 10.4, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 64.918, "latency": 18.56, "stderr": 1.629, "cost_per_test": 0.002351, "avg_input_tokens": 1493.9, "avg_output_tokens": 631.4, "avg_cache_read_tokens": 125.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.6, "median_query_seconds": 13.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 64.918, "latency": 76.495, "stderr": 1.629, "cost_per_test": 0.018879, "avg_input_tokens": 1434.5, "avg_output_tokens": 2859.6, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 76.5, "median_query_seconds": 65.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 64.802, "latency": 6.835, "stderr": 1.63, "cost_per_test": 0.001137, "avg_input_tokens": 1444.7, "avg_output_tokens": 1695.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 6.8, "median_query_seconds": 3.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 64.802, "latency": 38.306, "stderr": 1.63, "cost_per_test": 0.006637, "avg_input_tokens": 1402.5, "avg_output_tokens": 1326.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 38.3, "median_query_seconds": 26.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 48000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark": { "accuracy": 64.802, "latency": 41.977, "stderr": 1.63, "cost_per_test": 0.001586, "avg_input_tokens": 1418.3, "avg_output_tokens": 2288.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 42.0, "median_query_seconds": 35.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 64.569, "latency": 23.674, "stderr": 1.633, "cost_per_test": 0.009952, "avg_input_tokens": 1475.3, "avg_output_tokens": 1166.8, "avg_reasoning_tokens": 892.1, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 23.7, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 64.452, "latency": 15.54, "stderr": 1.634, "cost_per_test": 0.008411, "avg_input_tokens": 1377.8, "avg_output_tokens": 950.8, "avg_reasoning_tokens": 756.0, "avg_cache_read_tokens": 33.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 15.5, "median_query_seconds": 10.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 64.336, "latency": 6.738, "stderr": 1.635, "cost_per_test": null, "avg_input_tokens": 1544.1, "avg_output_tokens": 754.1, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 6.7, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 64.336, "latency": 10.524, "stderr": 1.635, "cost_per_test": 0.03284, "avg_input_tokens": 2230.4, "avg_output_tokens": 808.6, "avg_cache_read_tokens": 21.9, "avg_cache_write_tokens": 1256.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 10.5, "median_query_seconds": 7.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 64.336, "latency": 14.109, "stderr": 1.635, "cost_per_test": 0.039715, "avg_input_tokens": 2224.4, "avg_output_tokens": 1081.8, "avg_cache_read_tokens": 8.8, "avg_cache_write_tokens": 1269.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 14.1, "median_query_seconds": 11.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-4.5": { "accuracy": 64.336, "latency": 57.11, "stderr": 1.635, "cost_per_test": 0.002725, "avg_input_tokens": 1405.1, "avg_output_tokens": 858.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 57.1, "median_query_seconds": 53.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 64.103, "latency": 51.952, "stderr": 1.638, "cost_per_test": 0.025446, "avg_input_tokens": 1383.8, "avg_output_tokens": 1465.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 52.0, "median_query_seconds": 20.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 63.986, "latency": 8.361, "stderr": 1.639, "cost_per_test": 0.00449, "avg_input_tokens": 1364.8, "avg_output_tokens": 679.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 8.4, "median_query_seconds": 6.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 63.869, "latency": 12.323, "stderr": 1.64, "cost_per_test": 0.027074, "avg_input_tokens": 2224.4, "avg_output_tokens": 1251.6, "avg_cache_write_tokens": 2170.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 12.3, "median_query_seconds": 8.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 63.636, "latency": 12.744, "stderr": 1.642, "cost_per_test": 0.005283, "avg_input_tokens": 1410.0, "avg_output_tokens": 752.8, "avg_reasoning_tokens": 532.3, "avg_cache_read_tokens": 3.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 12.7, "median_query_seconds": 9.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 63.52, "latency": 3.526, "stderr": 1.643, "cost_per_test": 0.000229, "avg_input_tokens": 1483.3, "avg_output_tokens": 536.4, "avg_cache_read_tokens": 0.1, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 3.5, "median_query_seconds": 1.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 63.52, "latency": 10.487, "stderr": 1.643, "cost_per_test": 0.008451, "avg_input_tokens": 1364.8, "avg_output_tokens": 715.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 10.5, "median_query_seconds": 7.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 63.52, "latency": 10.874, "stderr": 1.643, "cost_per_test": 0.001987, "avg_input_tokens": 1383.8, "avg_output_tokens": 1425.5, "avg_reasoning_tokens": 1227.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 10.9, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 63.403, "latency": 9.631, "stderr": 1.644, "cost_per_test": 0.020493, "avg_input_tokens": 1315.1, "avg_output_tokens": 556.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 9.6, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 63.287, "latency": 26.006, "stderr": 1.646, "cost_per_test": 0.001529, "avg_input_tokens": 1469.8, "avg_output_tokens": 1718.1, "avg_reasoning_tokens": 1529.5, "avg_cache_read_tokens": 1403.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 26.0, "median_query_seconds": 11.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.6-max-preview": { "accuracy": 63.287, "latency": 42.169, "stderr": 1.646, "cost_per_test": 0.013772, "avg_input_tokens": 1437.3, "avg_output_tokens": 1526.1, "avg_reasoning_tokens": 1281.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 42.2, "median_query_seconds": 35.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 63.17, "latency": 4.35, "stderr": 1.647, "cost_per_test": 0.007123, "avg_input_tokens": 1383.8, "avg_output_tokens": 362.9, "avg_reasoning_tokens": 166.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 4.3, "median_query_seconds": 3.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 63.17, "latency": 9.373, "stderr": 1.647, "cost_per_test": 0.014257, "avg_input_tokens": 1593.2, "avg_output_tokens": 595.6, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 9.4, "median_query_seconds": 7.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 63.054, "latency": 3.81, "stderr": 1.648, "cost_per_test": 0.008149, "avg_input_tokens": 1441.8, "avg_output_tokens": 877.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 3.8, "median_query_seconds": 3.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 63.054, "latency": 24.022, "stderr": 1.648, "cost_per_test": 0.004427, "avg_input_tokens": 1383.8, "avg_output_tokens": 269.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 24.0, "median_query_seconds": 17.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 62.937, "latency": 32.815, "stderr": 1.649, "cost_per_test": 0.201351, "avg_input_tokens": 158904.3, "avg_output_tokens": 272.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 32.8, "median_query_seconds": 29.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 62.821, "latency": 3.975, "stderr": 1.65, "cost_per_test": 0.005077, "avg_input_tokens": 1365.8, "avg_output_tokens": 293.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 4.0, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 62.821, "latency": 5.149, "stderr": 1.65, "cost_per_test": 0.011121, "avg_input_tokens": 1441.4, "avg_output_tokens": 995.5, "avg_reasoning_tokens": 779.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 5.1, "median_query_seconds": 4.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 61.888, "latency": 46.794, "stderr": 1.658, "cost_per_test": 0.000868, "avg_input_tokens": 1451.8, "avg_output_tokens": 1083.2, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 46.8, "median_query_seconds": 45.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.1": { "accuracy": 61.772, "latency": 18.86, "stderr": 1.659, "cost_per_test": 0.005086, "avg_input_tokens": 1403.0, "avg_output_tokens": 1155.8, "avg_reasoning_tokens": 933.4, "avg_cache_read_tokens": 19.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.9, "median_query_seconds": 13.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 61.655, "latency": 10.304, "stderr": 1.66, "cost_per_test": 0.002217, "avg_input_tokens": 1358.1, "avg_output_tokens": 716.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 10.3, "median_query_seconds": 7.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 61.538, "latency": 8.564, "stderr": 1.661, "cost_per_test": 0.026875, "avg_input_tokens": 1383.8, "avg_output_tokens": 665.8, "avg_reasoning_tokens": 487.5, "avg_cache_read_tokens": 4.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 8.6, "median_query_seconds": 3.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 61.538, "latency": 11.934, "stderr": 1.661, "cost_per_test": 0.00108, "avg_input_tokens": 1532.3, "avg_output_tokens": 233.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 11.9, "median_query_seconds": 5.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 61.538, "latency": 44.668, "stderr": 1.661, "cost_per_test": 0.014516, "avg_input_tokens": 1385.6, "avg_output_tokens": 2994.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 44.7, "median_query_seconds": 21.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 61.305, "latency": 15.39, "stderr": 1.663, "cost_per_test": 0.001629, "avg_input_tokens": 1357.3, "avg_output_tokens": 942.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 15.4, "median_query_seconds": 8.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 61.072, "latency": 4.299, "stderr": 1.665, "cost_per_test": 0.008536, "avg_input_tokens": 1441.4, "avg_output_tokens": 849.9, "avg_reasoning_tokens": 652.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 4.3, "median_query_seconds": 3.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 60.956, "latency": 190.749, "stderr": 1.665, "cost_per_test": null, "avg_input_tokens": 1509.1, "avg_output_tokens": 911.0, "avg_reasoning_tokens": 606.3, "avg_cache_read_tokens": 35.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 190.7, "median_query_seconds": 189.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 60.839, "latency": 9.468, "stderr": 1.666, "cost_per_test": 0.000488, "avg_input_tokens": 1456.8, "avg_output_tokens": 1054.5, "avg_reasoning_tokens": 883.4, "avg_cache_read_tokens": 84.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 9.5, "median_query_seconds": 3.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 60.839, "latency": 11.462, "stderr": 1.666, "cost_per_test": 0.007209, "avg_input_tokens": 101.8, "avg_output_tokens": 268.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 11.5, "median_query_seconds": 11.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 60.839, "latency": 19.092, "stderr": 1.666, "cost_per_test": 0.012811, "avg_input_tokens": 1441.4, "avg_output_tokens": 827.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 19.1, "median_query_seconds": 12.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 60.839, "latency": 200.413, "stderr": 1.666, "cost_per_test": 0.006714, "avg_input_tokens": 1438.3, "avg_output_tokens": 2557.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 200.4, "median_query_seconds": 27.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 60.723, "latency": 13.893, "stderr": 1.667, "cost_per_test": 0.00112, "avg_input_tokens": 1438.3, "avg_output_tokens": 2439.3, "avg_reasoning_tokens": 2202.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 13.9, "median_query_seconds": 10.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 60.606, "latency": 12.18, "stderr": 1.668, "cost_per_test": 0.01402, "avg_input_tokens": 1604.2, "avg_output_tokens": 613.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 12.2, "median_query_seconds": 10.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 60.606, "latency": 28.533, "stderr": 1.668, "cost_per_test": 0.00541, "avg_input_tokens": 1437.3, "avg_output_tokens": 1563.9, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 28.5, "median_query_seconds": 24.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 60.373, "latency": 8.67, "stderr": 1.67, "cost_per_test": 0.000735, "avg_input_tokens": 1383.8, "avg_output_tokens": 366.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 8.7, "median_query_seconds": 6.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 60.373, "latency": 9.701, "stderr": 1.67, "cost_per_test": 0.007791, "avg_input_tokens": 1576.2, "avg_output_tokens": 204.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 9.7, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 60.373, "latency": 11.967, "stderr": 1.67, "cost_per_test": 0.00107, "avg_input_tokens": 1675.5, "avg_output_tokens": 488.4, "avg_reasoning_tokens": 262.1, "avg_cache_read_tokens": 193.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 12.0, "median_query_seconds": 10.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 60.373, "latency": 12.461, "stderr": 1.67, "cost_per_test": 0.001418, "avg_input_tokens": 1357.8, "avg_output_tokens": 217.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 12.5, "median_query_seconds": 10.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001": { "accuracy": 60.256, "latency": 5.204, "stderr": 1.671, "cost_per_test": 0.002564, "avg_input_tokens": 1576.6, "avg_output_tokens": 198.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 5.2, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 60.256, "latency": 7.227, "stderr": 1.671, "cost_per_test": 0.00105, "avg_input_tokens": 1338.9, "avg_output_tokens": 460.9, "avg_cache_read_tokens": 16.0, "avg_cache_write_tokens": 1322.9, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 7.2, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5-thinking": { "accuracy": 60.14, "latency": 31.137, "stderr": 1.672, "cost_per_test": 0.006603, "avg_input_tokens": 1403.0, "avg_output_tokens": 1625.1, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 31.1, "median_query_seconds": 24.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "arcee-ai/trinity-large-thinking": { "accuracy": 60.023, "latency": 9.333, "stderr": 1.672, "cost_per_test": 0.000879, "avg_input_tokens": 1330.7, "avg_output_tokens": 973.6, "avg_reasoning_tokens": 361.8, "avg_cache_read_tokens": 21.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 9.3, "median_query_seconds": 6.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Arcee-Ai", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 60.023, "latency": 15.358, "stderr": 1.672, "cost_per_test": 0.01138, "avg_input_tokens": 1437.3, "avg_output_tokens": 1038.3, "avg_reasoning_tokens": 827.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 15.4, "median_query_seconds": 12.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-4.7": { "accuracy": 60.023, "latency": 39.216, "stderr": 1.672, "cost_per_test": 0.00438, "avg_input_tokens": 1403.9, "avg_output_tokens": 1614.9, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 39.2, "median_query_seconds": 28.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 59.907, "latency": 5.561, "stderr": 1.673, "cost_per_test": 0.003066, "avg_input_tokens": 1432.5, "avg_output_tokens": 224.5, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 5.6, "median_query_seconds": 4.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 59.907, "latency": 16.774, "stderr": 1.673, "cost_per_test": 0.003881, "avg_input_tokens": 1421.0, "avg_output_tokens": 210.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 16.8, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max": { "accuracy": 59.79, "latency": 7.092, "stderr": 1.674, "cost_per_test": 0.002796, "avg_input_tokens": 1432.5, "avg_output_tokens": 179.5, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 7.1, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 59.79, "latency": 10.947, "stderr": 1.674, "cost_per_test": 0.01504, "avg_input_tokens": 1582.6, "avg_output_tokens": 686.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 10.9, "median_query_seconds": 9.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 18384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 59.79, "latency": 20.072, "stderr": 1.674, "cost_per_test": 0.035945, "avg_input_tokens": 70727.9, "avg_output_tokens": 471.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.1, "median_query_seconds": 15.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 59.79, "latency": 148.152, "stderr": 1.674, "cost_per_test": 0.003682, "avg_input_tokens": 1376.3, "avg_output_tokens": 196.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 148.2, "median_query_seconds": 139.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 59.674, "latency": 27.389, "stderr": 1.675, "cost_per_test": 0.006658, "avg_input_tokens": 1437.3, "avg_output_tokens": 1609.9, "avg_reasoning_tokens": 1390.1, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 27.4, "median_query_seconds": 22.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 59.557, "latency": 20.406, "stderr": 1.675, "cost_per_test": 0.00529, "avg_input_tokens": 1354.3, "avg_output_tokens": 673.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.4, "median_query_seconds": 16.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 59.557, "latency": 146.965, "stderr": 1.675, "cost_per_test": null, "avg_input_tokens": 1546.1, "avg_output_tokens": 754.0, "avg_reasoning_tokens": 464.8, "avg_cache_read_tokens": 61.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 147.0, "median_query_seconds": 156.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 59.441, "latency": 3.307, "stderr": 1.676, "cost_per_test": 0.000891, "avg_input_tokens": 1365.8, "avg_output_tokens": 215.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 3.3, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 59.441, "latency": 5.643, "stderr": 1.676, "cost_per_test": 0.00025, "avg_input_tokens": 1671.5, "avg_output_tokens": 463.5, "avg_reasoning_tokens": 230.1, "avg_cache_read_tokens": 826.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 5.6, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 59.441, "latency": 14.982, "stderr": 1.676, "cost_per_test": 0.005289, "avg_input_tokens": 1441.4, "avg_output_tokens": 200.6, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 15.0, "median_query_seconds": 12.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 59.207, "latency": 84.032, "stderr": 1.678, "cost_per_test": 0.007781, "avg_input_tokens": 1456.8, "avg_output_tokens": 1546.0, "avg_reasoning_tokens": 1372.4, "avg_cache_read_tokens": 83.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 84.0, "median_query_seconds": 33.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 59.091, "latency": 67.814, "stderr": 1.679, "cost_per_test": 0.003367, "avg_input_tokens": 1412.0, "avg_output_tokens": 1075.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 67.8, "median_query_seconds": 57.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 58.974, "latency": 32.257, "stderr": 1.679, "cost_per_test": 0.000678, "avg_input_tokens": 1451.8, "avg_output_tokens": 1922.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 32.3, "median_query_seconds": 28.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 58.858, "latency": 17.494, "stderr": 1.68, "cost_per_test": 0.003881, "avg_input_tokens": 1383.8, "avg_output_tokens": 1767.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 17.5, "median_query_seconds": 10.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 58.741, "latency": 6.717, "stderr": 1.681, "cost_per_test": 0.001386, "avg_input_tokens": 1357.8, "avg_output_tokens": 182.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 6.7, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 58.741, "latency": 35.19, "stderr": 1.681, "cost_per_test": 0.014194, "avg_input_tokens": 1604.3, "avg_output_tokens": 625.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 35.2, "median_query_seconds": 25.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 58.625, "latency": 9.317, "stderr": 1.681, "cost_per_test": 0.004717, "avg_input_tokens": 1604.2, "avg_output_tokens": 622.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 9.3, "median_query_seconds": 7.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 58.508, "latency": 5.2, "stderr": 1.682, "cost_per_test": 0.007941, "avg_input_tokens": 1576.3, "avg_output_tokens": 214.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 5.2, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-3": { "accuracy": 58.508, "latency": 5.488, "stderr": 1.682, "cost_per_test": 0.007219, "avg_input_tokens": 1305.6, "avg_output_tokens": 220.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 5.5, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 58.508, "latency": 27.393, "stderr": 1.682, "cost_per_test": 0.000497, "avg_input_tokens": 1542.6, "avg_output_tokens": 2100.2, "avg_cache_read_tokens": 7.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 27.4, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-4.6": { "accuracy": 58.159, "latency": 15.911, "stderr": 1.684, "cost_per_test": 0.003792, "avg_input_tokens": 1402.9, "avg_output_tokens": 1341.1, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 15.9, "median_query_seconds": 14.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 57.809, "latency": 0.047, "stderr": 1.686, "cost_per_test": 0.007112, "avg_input_tokens": 1576.3, "avg_output_tokens": 158.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.0, "median_query_seconds": 0.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 57.459, "latency": 2.557, "stderr": 1.688, "cost_per_test": 0.002608, "avg_input_tokens": 1441.4, "avg_output_tokens": 870.3, "avg_reasoning_tokens": 686.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 2.6, "median_query_seconds": 2.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-2-1212": { "accuracy": 57.226, "latency": 3.662, "stderr": 1.689, "cost_per_test": null, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3p1": { "accuracy": 56.876, "latency": 31.727, "stderr": 1.691, "cost_per_test": 0.000982, "avg_input_tokens": 1347.1, "avg_output_tokens": 135.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 31.7, "median_query_seconds": 29.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 56.76, "latency": 57.491, "stderr": 1.691, "cost_per_test": 0.00088, "avg_input_tokens": 1439.7, "avg_output_tokens": 179.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 57.5, "median_query_seconds": 59.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 56.643, "latency": 3.379, "stderr": 1.692, "cost_per_test": 0.001726, "avg_input_tokens": 1441.4, "avg_output_tokens": 910.1, "avg_reasoning_tokens": 728.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 3.4, "median_query_seconds": 2.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 56.41, "latency": 0.062, "stderr": 1.693, "cost_per_test": 0.001851, "avg_input_tokens": 1576.3, "avg_output_tokens": 147.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.1, "median_query_seconds": 0.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 56.41, "latency": 12.114, "stderr": 1.693, "cost_per_test": 0.004987, "avg_input_tokens": 1364.8, "avg_output_tokens": 792.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 12.1, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 56.177, "latency": 3.828, "stderr": 1.694, "cost_per_test": 0.005637, "avg_input_tokens": 1443.1, "avg_output_tokens": 203.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 3.8, "median_query_seconds": 2.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 56.177, "latency": 57.467, "stderr": 1.694, "cost_per_test": 0.000911, "avg_input_tokens": 1439.7, "avg_output_tokens": 191.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 57.5, "median_query_seconds": 61.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 55.828, "latency": 3.167, "stderr": 1.695, "cost_per_test": 0.000535, "avg_input_tokens": 1353.5, "avg_output_tokens": 269.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 3.2, "median_query_seconds": 2.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 55.478, "latency": 8.521, "stderr": 1.697, "cost_per_test": 0.002287, "avg_input_tokens": 1532.8, "avg_output_tokens": 1015.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 8.5, "median_query_seconds": 6.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 55.361, "latency": 30.593, "stderr": 1.697, "cost_per_test": 0.018845, "avg_input_tokens": 1544.1, "avg_output_tokens": 2203.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 30.6, "median_query_seconds": 24.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 55.245, "latency": 2.215, "stderr": 1.698, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 54.779, "latency": 1.657, "stderr": 1.699, "cost_per_test": 0.000864, "avg_input_tokens": 1421.8, "avg_output_tokens": 174.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 1.7, "median_query_seconds": 1.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 53.846, "latency": 57.443, "stderr": 1.702, "cost_per_test": 0.000212, "avg_input_tokens": 1439.7, "avg_output_tokens": 169.1, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 57.4, "median_query_seconds": 61.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 53.73, "latency": 59.381, "stderr": 1.702, "cost_per_test": 0.000222, "avg_input_tokens": 1439.7, "avg_output_tokens": 195.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 59.4, "median_query_seconds": 65.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 52.681, "latency": 2.476, "stderr": 1.705, "cost_per_test": 0.000367, "avg_input_tokens": 1353.5, "avg_output_tokens": 209.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 2.5, "median_query_seconds": 1.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 52.448, "latency": 8.023, "stderr": 1.705, "cost_per_test": 0.000173, "avg_input_tokens": 1511.4, "avg_output_tokens": 198.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 8.0, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 52.214, "latency": 0.012, "stderr": 1.705, "cost_per_test": 0.000268, "avg_input_tokens": 1384.8, "avg_output_tokens": 100.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.0, "median_query_seconds": 0.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 51.981, "latency": 16.96, "stderr": 1.706, "cost_per_test": 0.011202, "avg_input_tokens": 1532.5, "avg_output_tokens": 1629.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 17.0, "median_query_seconds": 12.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.0-pro-exp-02-05": { "accuracy": 51.166, "latency": 5.275, "stderr": 1.707, "cost_per_test": 0.002429, "avg_input_tokens": 1421.0, "avg_output_tokens": 130.6, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 5.3, "median_query_seconds": 1.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { "accuracy": 50.816, "latency": 0.012, "stderr": 1.707, "cost_per_test": 0.000288, "avg_input_tokens": 1432.8, "avg_output_tokens": 169.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.0, "median_query_seconds": 0.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 50.233, "latency": 0.073, "stderr": 1.707, "cost_per_test": 0.004761, "avg_input_tokens": 1384.8, "avg_output_tokens": 129.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.1, "median_query_seconds": 0.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 50.117, "latency": 7.516, "stderr": 1.707, "cost_per_test": 0.0044, "avg_input_tokens": 1525.0, "avg_output_tokens": 168.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 7.5, "median_query_seconds": 1.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 49.417, "latency": 46.214, "stderr": 1.707, "cost_per_test": 0.111403, "avg_input_tokens": 111053.4, "avg_output_tokens": 116.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 46.2, "median_query_seconds": 38.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 48.951, "latency": 3.869, "stderr": 1.707, "cost_per_test": 0.000183, "avg_input_tokens": 1365.8, "avg_output_tokens": 116.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 3.9, "median_query_seconds": 3.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { "accuracy": 48.252, "latency": 0.073, "stderr": 1.706, "cost_per_test": 0.001373, "avg_input_tokens": 1432.8, "avg_output_tokens": 127.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.1, "median_query_seconds": 0.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 46.853, "latency": 19.906, "stderr": 1.704, "cost_per_test": 0.03445, "avg_input_tokens": 171652.9, "avg_output_tokens": 239.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 19.9, "median_query_seconds": 17.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ai21labs/jamba-1.5-large": { "accuracy": 45.455, "latency": 3.138, "stderr": 1.7, "cost_per_test": 0.003786, "avg_input_tokens": 1525.0, "avg_output_tokens": 92.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 3.1, "median_query_seconds": 1.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 44.988, "latency": 1.553, "stderr": 1.698, "cost_per_test": 0.000358, "avg_input_tokens": 1525.0, "avg_output_tokens": 132.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 1.6, "median_query_seconds": 0.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 44.755, "latency": 173.546, "stderr": 1.698, "cost_per_test": 0.075006, "avg_input_tokens": 131170.6, "avg_output_tokens": 975.0, "turn_metrics": { "tasks_measured": 856, "avg_turns": 1.0, "avg_query_seconds": 173.5, "median_query_seconds": 143.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 43.473, "latency": 355.63, "stderr": 1.692, "cost_per_test": 0.073712, "avg_input_tokens": 131203.1, "avg_output_tokens": 141.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 355.6, "median_query_seconds": 161.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 40.676, "latency": 20.551, "stderr": 1.677, "cost_per_test": 0.082015, "avg_input_tokens": 65117.1, "avg_output_tokens": 123.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.6, "median_query_seconds": 16.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ai21labs/jamba-1.5-mini": { "accuracy": 39.627, "latency": 0.655, "stderr": 1.67, "cost_per_test": 0.000336, "avg_input_tokens": 1525.0, "avg_output_tokens": 76.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.7, "median_query_seconds": 0.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "google/gemini-1.5-flash-001": { "accuracy": 38.462, "latency": 3.353, "stderr": 1.661, "cost_per_test": 0.00014, "avg_input_tokens": 1440.8, "avg_output_tokens": 105.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 3.4, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 37.762, "latency": 13.387, "stderr": 1.655, "cost_per_test": 0.004915, "avg_input_tokens": 65117.1, "avg_output_tokens": 103.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 13.4, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 35.431, "latency": 18.434, "stderr": 1.633, "cost_per_test": 0.006552, "avg_input_tokens": 65117.1, "avg_output_tokens": 99.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.4, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "max_fitting_context": { "anthropic/claude-fable-5": { "accuracy": 77.04, "latency": 81.472, "stderr": 1.436, "cost_per_test": 3.734685, "avg_input_tokens": 289283.7, "avg_output_tokens": 2372.8, "avg_cache_write_tokens": 289282.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 81.5, "median_query_seconds": 46.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 75.991, "latency": 40.655, "stderr": 1.458, "cost_per_test": 1.839482, "avg_input_tokens": 289283.7, "avg_output_tokens": 2701.2, "avg_cache_read_tokens": 6272.6, "avg_cache_write_tokens": 283009.1, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 40.7, "median_query_seconds": 29.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 74.359, "latency": 39.377, "stderr": 1.491, "cost_per_test": 0.230768, "avg_input_tokens": 178685.1, "avg_output_tokens": 1867.8, "avg_reasoning_tokens": 1420.6, "avg_cache_read_tokens": 478.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 39.4, "median_query_seconds": 35.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 74.009, "latency": 41.511, "stderr": 1.497, "cost_per_test": 0.230884, "avg_input_tokens": 178528.1, "avg_output_tokens": 1818.2, "avg_reasoning_tokens": 1253.9, "avg_cache_read_tokens": 2.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 41.5, "median_query_seconds": 26.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 72.727, "latency": 64.691, "stderr": 1.52, "cost_per_test": 0.122828, "avg_input_tokens": 182139.5, "avg_output_tokens": 1267.2, "avg_reasoning_tokens": 796.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 64.7, "median_query_seconds": 56.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 70.629, "latency": 198.506, "stderr": 1.555, "cost_per_test": 0.565658, "avg_input_tokens": 174916.2, "avg_output_tokens": 3215.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 198.5, "median_query_seconds": 197.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 70.396, "latency": 45.52, "stderr": 1.558, "cost_per_test": 0.052717, "avg_input_tokens": 170122.6, "avg_output_tokens": 1589.3, "avg_cache_read_tokens": 944.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 45.5, "median_query_seconds": 27.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.5": { "accuracy": 70.047, "latency": 25.327, "stderr": 1.564, "cost_per_test": 0.902658, "avg_input_tokens": 172644.9, "avg_output_tokens": 1314.4, "avg_reasoning_tokens": 1024.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 25.3, "median_query_seconds": 15.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 69.814, "latency": 26.189, "stderr": 1.567, "cost_per_test": 1.847531, "avg_input_tokens": 289283.7, "avg_output_tokens": 1705.1, "avg_cache_read_tokens": 542.3, "avg_cache_write_tokens": 288740.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 26.2, "median_query_seconds": 22.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 69.697, "latency": 31.268, "stderr": 1.569, "cost_per_test": 0.105284, "avg_input_tokens": 171667.5, "avg_output_tokens": 974.3, "avg_cache_read_tokens": 114.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 31.3, "median_query_seconds": 25.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 69.114, "latency": 23.105, "stderr": 1.577, "cost_per_test": 1.105176, "avg_input_tokens": 286151.6, "avg_output_tokens": 2140.6, "avg_cache_write_tokens": 286150.6, "turn_metrics": { "tasks_measured": 843, "avg_turns": 1.0, "avg_query_seconds": 23.1, "median_query_seconds": 15.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 68.765, "latency": 38.739, "stderr": 1.582, "cost_per_test": 0.316319, "avg_input_tokens": 171689.7, "avg_output_tokens": 1575.8, "avg_reasoning_tokens": 1155.0, "avg_cache_read_tokens": 1.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 38.7, "median_query_seconds": 24.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.2": { "accuracy": 68.415, "latency": 34.665, "stderr": 1.587, "cost_per_test": 0.262089, "avg_input_tokens": 182435.3, "avg_output_tokens": 1518.3, "avg_reasoning_tokens": 1164.8, "avg_cache_read_tokens": 0.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 34.7, "median_query_seconds": 27.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 68.415, "latency": 132.479, "stderr": 1.587, "cost_per_test": 0.109485, "avg_input_tokens": 175868.1, "avg_output_tokens": 1571.9, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 132.5, "median_query_seconds": 114.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.6-max-preview": { "accuracy": 68.182, "latency": 47.888, "stderr": 1.59, "cost_per_test": 0.351906, "avg_input_tokens": 168657.5, "avg_output_tokens": 1697.5, "avg_reasoning_tokens": 1364.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 47.9, "median_query_seconds": 43.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 67.832, "latency": 24.506, "stderr": 1.595, "cost_per_test": 1.760496, "avg_input_tokens": 285088.6, "avg_output_tokens": 1206.3, "avg_cache_read_tokens": 1642.3, "avg_cache_write_tokens": 283441.4, "turn_metrics": { "tasks_measured": 838, "avg_turns": 1.0, "avg_query_seconds": 24.5, "median_query_seconds": 19.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 67.599, "latency": 23.995, "stderr": 1.598, "cost_per_test": 0.173886, "avg_input_tokens": 168606.1, "avg_output_tokens": 1495.7, "avg_cache_read_tokens": 937.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 24.0, "median_query_seconds": 15.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.5": { "accuracy": 67.483, "latency": 109.542, "stderr": 1.599, "cost_per_test": 0.524857, "avg_input_tokens": 175115.9, "avg_output_tokens": 1096.3, "avg_reasoning_tokens": 817.3, "avg_cache_read_tokens": 392.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 109.5, "median_query_seconds": 108.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 67.249, "latency": 103.491, "stderr": 1.602, "cost_per_test": 0.171139, "avg_input_tokens": 172296.6, "avg_output_tokens": 1866.3, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 9.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 103.5, "median_query_seconds": 93.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 48000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 67.249, "latency": 226.284, "stderr": 1.602, "cost_per_test": 0.133024, "avg_input_tokens": 184341.0, "avg_output_tokens": 3565.6, "turn_metrics": { "tasks_measured": 853, "avg_turns": 1.0, "avg_query_seconds": 226.3, "median_query_seconds": 62.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 67.133, "latency": 22.607, "stderr": 1.604, "cost_per_test": null, "avg_input_tokens": 199641.3, "avg_output_tokens": 1337.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 22.6, "median_query_seconds": 20.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 67.133, "latency": 28.963, "stderr": 1.604, "cost_per_test": 0.9616, "avg_input_tokens": 149217.2, "avg_output_tokens": 1164.8, "turn_metrics": { "tasks_measured": 857, "avg_turns": 1.0, "avg_query_seconds": 29.0, "median_query_seconds": 22.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 67.016, "latency": 21.418, "stderr": 1.605, "cost_per_test": 0.377404, "avg_input_tokens": 186929.3, "avg_output_tokens": 295.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 21.4, "median_query_seconds": 18.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 66.9, "latency": 28.935, "stderr": 1.607, "cost_per_test": 0.387473, "avg_input_tokens": 187158.4, "avg_output_tokens": 1096.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 28.9, "median_query_seconds": 22.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 66.667, "latency": 12.357, "stderr": 1.609, "cost_per_test": 0.29559, "avg_input_tokens": 187158.4, "avg_output_tokens": 1650.2, "avg_reasoning_tokens": 1350.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 12.4, "median_query_seconds": 9.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 66.317, "latency": 9.507, "stderr": 1.614, "cost_per_test": 0.369227, "avg_input_tokens": 179821.1, "avg_output_tokens": 837.3, "avg_reasoning_tokens": 556.9, "avg_cache_read_tokens": 256.9, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 9.5, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 66.317, "latency": 139.915, "stderr": 1.614, "cost_per_test": 0.15991, "avg_input_tokens": 153522.2, "avg_output_tokens": 2162.3, "avg_reasoning_tokens": 1824.9, "avg_cache_read_tokens": 664.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 139.9, "median_query_seconds": 39.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.6": { "accuracy": 66.2, "latency": 33.17, "stderr": 1.615, "cost_per_test": 0.356004, "avg_input_tokens": 175115.9, "avg_output_tokens": 1235.5, "avg_reasoning_tokens": 999.4, "avg_cache_read_tokens": 1093.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 33.2, "median_query_seconds": 28.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 66.2, "latency": 34.411, "stderr": 1.615, "cost_per_test": 0.002636, "avg_input_tokens": 179004.5, "avg_output_tokens": 2251.7, "avg_reasoning_tokens": 2015.0, "avg_cache_read_tokens": 178939.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 34.4, "median_query_seconds": 21.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 66.084, "latency": 26.038, "stderr": 1.616, "cost_per_test": 0.620313, "avg_input_tokens": 161235.1, "avg_output_tokens": 1244.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 26.0, "median_query_seconds": 19.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 65.734, "latency": 26.109, "stderr": 1.62, "cost_per_test": 0.473648, "avg_input_tokens": 185375.8, "avg_output_tokens": 1361.2, "avg_reasoning_tokens": 1066.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 26.1, "median_query_seconds": 20.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 65.734, "latency": 57.235, "stderr": 1.62, "cost_per_test": 0.019801, "avg_input_tokens": 185376.8, "avg_output_tokens": 3158.2, "avg_reasoning_tokens": 2797.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 57.2, "median_query_seconds": 55.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 65.618, "latency": 23.184, "stderr": 1.622, "cost_per_test": 0.024271, "avg_input_tokens": 109.5, "avg_output_tokens": 948.9, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 23.2, "median_query_seconds": 20.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 65.618, "latency": 60.398, "stderr": 1.622, "cost_per_test": 0.381993, "avg_input_tokens": 185413.8, "avg_output_tokens": 1860.9, "avg_reasoning_tokens": 1440.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 60.4, "median_query_seconds": 41.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 65.385, "latency": 7.954, "stderr": 1.624, "cost_per_test": 0.291436, "avg_input_tokens": 187158.4, "avg_output_tokens": 1426.5, "avg_reasoning_tokens": 1122.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 8.0, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 65.152, "latency": 18.943, "stderr": 1.627, "cost_per_test": 1.060797, "avg_input_tokens": 179826.0, "avg_output_tokens": 1475.2, "avg_reasoning_tokens": 1225.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.9, "median_query_seconds": 10.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 64.918, "latency": 72.418, "stderr": 1.629, "cost_per_test": 0.483226, "avg_input_tokens": 179823.4, "avg_output_tokens": 2244.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 72.4, "median_query_seconds": 54.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 64.918, "latency": 109.112, "stderr": 1.629, "cost_per_test": 0.107726, "avg_input_tokens": 168657.5, "avg_output_tokens": 1814.4, "avg_reasoning_tokens": 1539.4, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 109.1, "median_query_seconds": 109.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 64.802, "latency": 33.846, "stderr": 1.63, "cost_per_test": 0.000351, "avg_input_tokens": 1300.9, "avg_output_tokens": 181.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 33.8, "median_query_seconds": 23.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 64.103, "latency": 12.831, "stderr": 1.638, "cost_per_test": 0.000347, "avg_input_tokens": 1300.0, "avg_output_tokens": 174.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 12.8, "median_query_seconds": 9.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 64.103, "latency": 19.928, "stderr": 1.638, "cost_per_test": 0.03887, "avg_input_tokens": 179821.2, "avg_output_tokens": 2421.3, "avg_reasoning_tokens": 2144.6, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 19.9, "median_query_seconds": 10.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 63.869, "latency": 14.006, "stderr": 1.64, "cost_per_test": 0.016663, "avg_input_tokens": 1383.8, "avg_output_tokens": 1017.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 14.0, "median_query_seconds": 9.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 63.753, "latency": 53.318, "stderr": 1.641, "cost_per_test": 0.001232, "avg_input_tokens": 1526.2, "avg_output_tokens": 327.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 53.3, "median_query_seconds": 40.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 63.52, "latency": 7.32, "stderr": 1.643, "cost_per_test": 0.003928, "avg_input_tokens": 1441.4, "avg_output_tokens": 1069.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 7.3, "median_query_seconds": 4.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 63.403, "latency": 48.958, "stderr": 1.644, "cost_per_test": 0.181071, "avg_input_tokens": 141872.7, "avg_output_tokens": 373.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 49.0, "median_query_seconds": 42.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 63.054, "latency": 79.933, "stderr": 1.648, "cost_per_test": 0.316589, "avg_input_tokens": 178991.5, "avg_output_tokens": 2389.5, "avg_reasoning_tokens": 2162.8, "avg_cache_read_tokens": 1988.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 79.9, "median_query_seconds": 48.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 62.937, "latency": 43.066, "stderr": 1.649, "cost_per_test": 0.097946, "avg_input_tokens": 185375.8, "avg_output_tokens": 1752.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 43.1, "median_query_seconds": 36.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-0709": { "accuracy": 62.821, "latency": 43.442, "stderr": 1.65, "cost_per_test": 0.345687, "avg_input_tokens": 107850.4, "avg_output_tokens": 1475.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 43.4, "median_query_seconds": 38.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 62.704, "latency": 22.531, "stderr": 1.651, "cost_per_test": 0.02548, "avg_input_tokens": 178991.5, "avg_output_tokens": 1503.6, "avg_reasoning_tokens": 1288.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 22.5, "median_query_seconds": 18.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 62.354, "latency": 4.541, "stderr": 1.654, "cost_per_test": 0.059711, "avg_input_tokens": 187158.4, "avg_output_tokens": 1425.3, "avg_reasoning_tokens": 1174.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 4.5, "median_query_seconds": 3.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 62.354, "latency": 16.413, "stderr": 1.654, "cost_per_test": 0.076169, "avg_input_tokens": 107960.3, "avg_output_tokens": 109154.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 16.4, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 62.121, "latency": 12.158, "stderr": 1.656, "cost_per_test": 0.035271, "avg_input_tokens": 172644.9, "avg_output_tokens": 593.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 12.2, "median_query_seconds": 9.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 62.121, "latency": 15.783, "stderr": 1.656, "cost_per_test": 0.351139, "avg_input_tokens": 171616.1, "avg_output_tokens": 1317.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 857, "avg_turns": 1.0, "avg_query_seconds": 15.8, "median_query_seconds": 14.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 61.538, "latency": 16.257, "stderr": 1.661, "cost_per_test": 0.483633, "avg_input_tokens": 160864.0, "avg_output_tokens": 296.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 16.3, "median_query_seconds": 14.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-5-thinking": { "accuracy": 61.538, "latency": 104.077, "stderr": 1.661, "cost_per_test": 0.123859, "avg_input_tokens": 114584.4, "avg_output_tokens": 2898.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 104.1, "median_query_seconds": 83.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 61.422, "latency": 28.107, "stderr": 1.662, "cost_per_test": 0.081304, "avg_input_tokens": 186287.1, "avg_output_tokens": 772.1, "avg_reasoning_tokens": 408.1, "avg_cache_read_tokens": 932.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 28.1, "median_query_seconds": 24.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 61.072, "latency": 27.456, "stderr": 1.665, "cost_per_test": 0.464498, "avg_input_tokens": 149301.0, "avg_output_tokens": 1106.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 27.5, "median_query_seconds": 23.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 61.072, "latency": 58.363, "stderr": 1.665, "cost_per_test": 0.291221, "avg_input_tokens": 181740.8, "avg_output_tokens": 2481.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 58.4, "median_query_seconds": 60.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 61.072, "latency": 175.462, "stderr": 1.665, "cost_per_test": 0.175312, "avg_input_tokens": 138294.9, "avg_output_tokens": 244.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 175.5, "median_query_seconds": 147.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 60.956, "latency": 66.943, "stderr": 1.665, "cost_per_test": 0.146241, "avg_input_tokens": 172644.9, "avg_output_tokens": 3723.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 66.9, "median_query_seconds": 36.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 60.839, "latency": 15.579, "stderr": 1.666, "cost_per_test": 0.261029, "avg_input_tokens": 127935.4, "avg_output_tokens": 644.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 15.6, "median_query_seconds": 11.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 60.723, "latency": 5.786, "stderr": 1.667, "cost_per_test": 0.011333, "avg_input_tokens": 1296.8, "avg_output_tokens": 194.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 5.8, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 60.606, "latency": 4.003, "stderr": 1.668, "cost_per_test": 0.048652, "avg_input_tokens": 187158.4, "avg_output_tokens": 1241.3, "avg_reasoning_tokens": 1005.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 4.0, "median_query_seconds": 3.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 60.606, "latency": 67.517, "stderr": 1.668, "cost_per_test": 0.056863, "avg_input_tokens": 187221.6, "avg_output_tokens": 278.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 67.5, "median_query_seconds": 66.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 60.49, "latency": 69.3, "stderr": 1.669, "cost_per_test": 0.056875, "avg_input_tokens": 187221.6, "avg_output_tokens": 283.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 69.3, "median_query_seconds": 69.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 60.49, "latency": 93.929, "stderr": 1.669, "cost_per_test": 0.039253, "avg_input_tokens": 148533.7, "avg_output_tokens": 1059.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 93.9, "median_query_seconds": 86.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 60.49, "latency": 351.337, "stderr": 1.669, "cost_per_test": 0.463634, "avg_input_tokens": 149301.0, "avg_output_tokens": 1048.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 351.3, "median_query_seconds": 102.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 60.14, "latency": 27.898, "stderr": 1.672, "cost_per_test": 0.026197, "avg_input_tokens": 186283.1, "avg_output_tokens": 709.0, "avg_reasoning_tokens": 356.9, "avg_cache_read_tokens": 589.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 27.9, "median_query_seconds": 26.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 60.023, "latency": 18.36, "stderr": 1.672, "cost_per_test": 0.166751, "avg_input_tokens": 161216.0, "avg_output_tokens": 1146.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.4, "median_query_seconds": 16.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 60.023, "latency": 20.97, "stderr": 1.672, "cost_per_test": 0.013642, "avg_input_tokens": 178551.8, "avg_output_tokens": 1137.5, "avg_cache_read_tokens": 1.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 21.0, "median_query_seconds": 20.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "meta/muse_spark": { "accuracy": 60.023, "latency": 38.935, "stderr": 1.672, "cost_per_test": 0.022947, "avg_input_tokens": 145503.2, "avg_output_tokens": 1869.9, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 38.9, "median_query_seconds": 33.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 60.023, "latency": 70.079, "stderr": 1.672, "cost_per_test": 0.01883, "avg_input_tokens": 187221.6, "avg_output_tokens": 270.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 70.1, "median_query_seconds": 68.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 59.907, "latency": 70.729, "stderr": 1.673, "cost_per_test": 0.018892, "avg_input_tokens": 187221.6, "avg_output_tokens": 425.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 70.7, "median_query_seconds": 69.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 59.557, "latency": 14.069, "stderr": 1.675, "cost_per_test": 0.043303, "avg_input_tokens": 113209.5, "avg_output_tokens": 712.5, "avg_cache_read_tokens": 16.0, "avg_cache_write_tokens": 113193.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 14.1, "median_query_seconds": 13.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 59.207, "latency": 32.135, "stderr": 1.678, "cost_per_test": 0.452639, "avg_input_tokens": 145422.4, "avg_output_tokens": 1091.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 32.1, "median_query_seconds": 32.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 18384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-4-5-20251001": { "accuracy": 58.858, "latency": 10.123, "stderr": 1.68, "cost_per_test": 0.150828, "avg_input_tokens": 149273.0, "avg_output_tokens": 311.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 10.1, "median_query_seconds": 9.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 58.858, "latency": 15.178, "stderr": 1.68, "cost_per_test": 0.0018, "avg_input_tokens": 1350.6, "avg_output_tokens": 622.2, "turn_metrics": { "tasks_measured": 857, "avg_turns": 1.0, "avg_query_seconds": 15.2, "median_query_seconds": 10.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 58.392, "latency": 122.216, "stderr": 1.683, "cost_per_test": 0.06934, "avg_input_tokens": 114178.1, "avg_output_tokens": 2013.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 122.2, "median_query_seconds": 105.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 57.809, "latency": 11.897, "stderr": 1.686, "cost_per_test": 0.169274, "avg_input_tokens": 133179.0, "avg_output_tokens": 280.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 11.9, "median_query_seconds": 10.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-3": { "accuracy": 57.11, "latency": 49.7, "stderr": 1.69, "cost_per_test": 0.317909, "avg_input_tokens": 104281.4, "avg_output_tokens": 337.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 49.7, "median_query_seconds": 54.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 56.993, "latency": 298.129, "stderr": 1.69, "cost_per_test": null, "avg_input_tokens": 182792.6, "avg_output_tokens": 1078.3, "avg_reasoning_tokens": 679.0, "avg_cache_read_tokens": 1767.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 298.1, "median_query_seconds": 264.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 56.876, "latency": 20.714, "stderr": 1.691, "cost_per_test": 0.003627, "avg_input_tokens": 1403.4, "avg_output_tokens": 187.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.7, "median_query_seconds": 18.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 56.76, "latency": 21.903, "stderr": 1.691, "cost_per_test": 0.045262, "avg_input_tokens": 113385.5, "avg_output_tokens": 1147.6, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 21.9, "median_query_seconds": 17.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 56.061, "latency": 28.918, "stderr": 1.694, "cost_per_test": 0.260856, "avg_input_tokens": 127934.4, "avg_output_tokens": 623.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 28.9, "median_query_seconds": 25.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 55.828, "latency": 20.701, "stderr": 1.695, "cost_per_test": 0.032067, "avg_input_tokens": 104282.4, "avg_output_tokens": 1565.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.7, "median_query_seconds": 17.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-4.5": { "accuracy": 55.478, "latency": 136.325, "stderr": 1.697, "cost_per_test": 0.069957, "avg_input_tokens": 113914.2, "avg_output_tokens": 955.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 136.3, "median_query_seconds": 97.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 55.012, "latency": 8.221, "stderr": 1.698, "cost_per_test": 0.051823, "avg_input_tokens": 127935.4, "avg_output_tokens": 405.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 8.2, "median_query_seconds": 7.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 55.012, "latency": 27.127, "stderr": 1.698, "cost_per_test": 0.144774, "avg_input_tokens": 127934.4, "avg_output_tokens": 919.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 27.1, "median_query_seconds": 22.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 54.895, "latency": 5.336, "stderr": 1.699, "cost_per_test": 0.000945, "avg_input_tokens": 1348.9, "avg_output_tokens": 112.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 5.3, "median_query_seconds": 3.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 54.429, "latency": 277.907, "stderr": 1.7, "cost_per_test": null, "avg_input_tokens": 182935.4, "avg_output_tokens": 974.5, "avg_reasoning_tokens": 578.8, "avg_cache_read_tokens": 20.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 277.9, "median_query_seconds": 248.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 54.312, "latency": 28.337, "stderr": 1.701, "cost_per_test": 0.041668, "avg_input_tokens": 106957.2, "avg_output_tokens": 1315.2, "turn_metrics": { "tasks_measured": 855, "avg_turns": 1.0, "avg_query_seconds": 28.3, "median_query_seconds": 21.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 54.079, "latency": 13.932, "stderr": 1.701, "cost_per_test": 0.07579, "avg_input_tokens": 107970.3, "avg_output_tokens": 108391.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 13.9, "median_query_seconds": 10.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 53.263, "latency": 0.049, "stderr": 1.703, "cost_per_test": 0.486092, "avg_input_tokens": 161003.4, "avg_output_tokens": 205.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.0, "median_query_seconds": 0.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 53.03, "latency": 33.801, "stderr": 1.704, "cost_per_test": 0.001714, "avg_input_tokens": 1420.5, "avg_output_tokens": 98.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 33.8, "median_query_seconds": 23.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 52.681, "latency": 116.245, "stderr": 1.705, "cost_per_test": 0.177377, "avg_input_tokens": 146781.8, "avg_output_tokens": 206.4, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 116.2, "median_query_seconds": 120.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 52.681, "latency": 449.724, "stderr": 1.705, "cost_per_test": 0.452376, "avg_input_tokens": 149273.0, "avg_output_tokens": 303.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 449.7, "median_query_seconds": 82.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 52.331, "latency": 20.232, "stderr": 1.705, "cost_per_test": 0.040537, "avg_input_tokens": 132912.0, "avg_output_tokens": 265.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.2, "median_query_seconds": 11.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 52.098, "latency": 14.801, "stderr": 1.705, "cost_per_test": 0.031819, "avg_input_tokens": 104282.4, "avg_output_tokens": 1069.1, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 14.8, "median_query_seconds": 13.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 51.981, "latency": 41.001, "stderr": 1.706, "cost_per_test": 0.017607, "avg_input_tokens": 112982.6, "avg_output_tokens": 1133.6, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 41.0, "median_query_seconds": 40.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 51.865, "latency": 59.671, "stderr": 1.706, "cost_per_test": 0.009675, "avg_input_tokens": 181553.3, "avg_output_tokens": 3442.8, "avg_cache_read_tokens": 2272.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 59.7, "median_query_seconds": 40.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 50.932, "latency": 66.39, "stderr": 1.707, "cost_per_test": 0.098456, "avg_input_tokens": 109107.4, "avg_output_tokens": 288.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 66.4, "median_query_seconds": 54.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 50.233, "latency": 83.286, "stderr": 1.707, "cost_per_test": 0.166784, "avg_input_tokens": 120691.8, "avg_output_tokens": 712.9, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 83.3, "median_query_seconds": 82.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "arcee-ai/trinity-large-thinking": { "accuracy": 49.184, "latency": 16.74, "stderr": 1.707, "cost_per_test": 0.041363, "avg_input_tokens": 162893.4, "avg_output_tokens": 769.4, "avg_reasoning_tokens": 398.0, "avg_cache_read_tokens": 278.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 16.7, "median_query_seconds": 14.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Arcee-Ai", "harness": null }, "zai/glm-4.6": { "accuracy": 49.184, "latency": 155.354, "stderr": 1.707, "cost_per_test": 0.089158, "avg_input_tokens": 143678.4, "avg_output_tokens": 2013.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 155.4, "median_query_seconds": 73.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/deepseek-v3p1": { "accuracy": 48.252, "latency": 96.378, "stderr": 1.706, "cost_per_test": 0.075218, "avg_input_tokens": 133582.8, "avg_output_tokens": 245.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 96.4, "median_query_seconds": 91.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 47.669, "latency": 50.565, "stderr": 1.705, "cost_per_test": 0.098432, "avg_input_tokens": 109107.4, "avg_output_tokens": 261.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 50.6, "median_query_seconds": 41.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 46.97, "latency": 26.239, "stderr": 1.704, "cost_per_test": 0.007679, "avg_input_tokens": 99224.3, "avg_output_tokens": 2443.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 26.2, "median_query_seconds": 24.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 45.221, "latency": 6.636, "stderr": 1.699, "cost_per_test": 0.028134, "avg_input_tokens": 126867.9, "avg_output_tokens": 253.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 6.6, "median_query_seconds": 6.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-2-1212": { "accuracy": 45.221, "latency": 241.891, "stderr": 1.699, "cost_per_test": 0.216047, "avg_input_tokens": 106884.4, "avg_output_tokens": 227.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 241.9, "median_query_seconds": 235.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 44.755, "latency": 0.009, "stderr": 1.698, "cost_per_test": 0.129482, "avg_input_tokens": 161003.4, "avg_output_tokens": 169.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.0, "median_query_seconds": 0.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 44.522, "latency": 43.687, "stderr": 1.697, "cost_per_test": 0.042618, "avg_input_tokens": 105366.7, "avg_output_tokens": 235.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 43.7, "median_query_seconds": 34.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 44.056, "latency": 441.206, "stderr": 1.695, "cost_per_test": 0.205935, "avg_input_tokens": 171016.8, "avg_output_tokens": 119.1, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 441.2, "median_query_seconds": 294.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 43.124, "latency": 20.149, "stderr": 1.691, "cost_per_test": 0.023152, "avg_input_tokens": 126867.9, "avg_output_tokens": 535.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.1, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 43.007, "latency": 96.844, "stderr": 1.69, "cost_per_test": 0.216228, "avg_input_tokens": 105358.5, "avg_output_tokens": 1923.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 96.8, "median_query_seconds": 77.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 42.657, "latency": 10.92, "stderr": 1.688, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 41.958, "latency": 27.747, "stderr": 1.685, "cost_per_test": 0.445899, "avg_input_tokens": 177470.4, "avg_output_tokens": 222.3, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 27.7, "median_query_seconds": 27.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 41.026, "latency": 0.055, "stderr": 1.679, "cost_per_test": 0.017023, "avg_input_tokens": 112931.0, "avg_output_tokens": 138.5, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.1, "median_query_seconds": 0.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 40.443, "latency": 73.047, "stderr": 1.675, "cost_per_test": 0.229417, "avg_input_tokens": 183012.6, "avg_output_tokens": 130.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 73.0, "median_query_seconds": 58.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 38.928, "latency": 58.446, "stderr": 1.665, "cost_per_test": 0.013757, "avg_input_tokens": 183012.6, "avg_output_tokens": 103.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 58.4, "median_query_seconds": 36.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.0-pro-exp-02-05": { "accuracy": 38.695, "latency": 35.856, "stderr": 1.663, "cost_per_test": 0.229335, "avg_input_tokens": 183013.6, "avg_output_tokens": 113.6, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 35.9, "median_query_seconds": 30.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 38.462, "latency": 5.948, "stderr": 1.661, "cost_per_test": 0.012861, "avg_input_tokens": 127935.4, "avg_output_tokens": 169.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 5.9, "median_query_seconds": 5.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 38.462, "latency": 12.923, "stderr": 1.661, "cost_per_test": 0.007811, "avg_input_tokens": 102718.7, "avg_output_tokens": 356.6, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 12.9, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 36.597, "latency": 18.965, "stderr": 1.644, "cost_per_test": 0.053106, "avg_input_tokens": 105354.1, "avg_output_tokens": 327.6, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 19.0, "median_query_seconds": 17.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 36.364, "latency": 0.057, "stderr": 1.642, "cost_per_test": 0.283471, "avg_input_tokens": 112931.0, "avg_output_tokens": 114.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.1, "median_query_seconds": 0.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ai21labs/jamba-1.5-large": { "accuracy": 36.364, "latency": 19.187, "stderr": 1.642, "cost_per_test": 0.304754, "avg_input_tokens": 152027.9, "avg_output_tokens": 87.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 19.2, "median_query_seconds": 20.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 35.431, "latency": 40.708, "stderr": 1.633, "cost_per_test": 0.306016, "avg_input_tokens": 152027.9, "avg_output_tokens": 245.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 40.7, "median_query_seconds": 22.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 33.566, "latency": 57.794, "stderr": 1.612, "cost_per_test": 0.155768, "avg_input_tokens": 137725.6, "avg_output_tokens": 970.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 57.8, "median_query_seconds": 52.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 32.051, "latency": 7.8, "stderr": 1.593, "cost_per_test": 0.03048, "avg_input_tokens": 152027.9, "avg_output_tokens": 186.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 7.8, "median_query_seconds": 6.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "together/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { "accuracy": 30.653, "latency": 0.046, "stderr": 1.574, "cost_per_test": 0.100898, "avg_input_tokens": 114550.0, "avg_output_tokens": 106.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.0, "median_query_seconds": 0.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "ai21labs/jamba-1.5-mini": { "accuracy": 30.07, "latency": 4.221, "stderr": 1.566, "cost_per_test": 0.030431, "avg_input_tokens": 152027.9, "avg_output_tokens": 62.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 4.2, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 30.07, "latency": 53.247, "stderr": 1.566, "cost_per_test": 0.018339, "avg_input_tokens": 183012.6, "avg_output_tokens": 93.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 53.2, "median_query_seconds": 25.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-1.5-flash-001": { "accuracy": 25.991, "latency": 0.055, "stderr": 1.497, "cost_per_test": 0.014092, "avg_input_tokens": 187569.6, "avg_output_tokens": 80.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.1, "median_query_seconds": 0.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { "accuracy": 25.641, "latency": 0.049, "stderr": 1.491, "cost_per_test": 0.020359, "avg_input_tokens": 112838.1, "avg_output_tokens": 269.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.0, "median_query_seconds": 0.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-4.7": { "accuracy": 17.949, "latency": 93.697, "stderr": 1.31, "cost_per_test": 0.067835, "avg_input_tokens": 122192.8, "avg_output_tokens": 3396.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 93.7, "median_query_seconds": 69.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null } }, "shared_max_context": { "anthropic/claude-opus-5": { "accuracy": 74.942, "latency": 29.037, "stderr": 1.479, "cost_per_test": 0.696945, "avg_input_tokens": 102472.5, "avg_output_tokens": 2259.8, "avg_cache_write_tokens": 102470.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 29.0, "median_query_seconds": 18.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 73.077, "latency": 25.409, "stderr": 1.514, "cost_per_test": 0.087456, "avg_input_tokens": 63493.4, "avg_output_tokens": 1954.1, "avg_reasoning_tokens": 1507.7, "avg_cache_read_tokens": 196.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 25.4, "median_query_seconds": 21.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 72.727, "latency": 31.073, "stderr": 1.52, "cost_per_test": 0.086787, "avg_input_tokens": 63336.4, "avg_output_tokens": 1793.2, "avg_reasoning_tokens": 1242.6, "avg_cache_read_tokens": 3.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 31.1, "median_query_seconds": 15.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 72.611, "latency": 30.106, "stderr": 1.522, "cost_per_test": 0.056251, "avg_input_tokens": 64795.5, "avg_output_tokens": 1287.8, "avg_reasoning_tokens": 844.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 30.1, "median_query_seconds": 22.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 71.562, "latency": 58.898, "stderr": 1.54, "cost_per_test": 1.387598, "avg_input_tokens": 102472.5, "avg_output_tokens": 2133.9, "avg_cache_write_tokens": 102471.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 58.9, "median_query_seconds": 33.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 71.096, "latency": 10.442, "stderr": 1.548, "cost_per_test": 0.043356, "avg_input_tokens": 61104.3, "avg_output_tokens": 62269.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 10.4, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark": { "accuracy": 70.513, "latency": 39.26, "stderr": 1.557, "cost_per_test": 0.010715, "avg_input_tokens": 63348.2, "avg_output_tokens": 2021.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 39.3, "median_query_seconds": 31.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 70.163, "latency": 69.331, "stderr": 1.562, "cost_per_test": 0.043611, "avg_input_tokens": 64717.5, "avg_output_tokens": 1696.9, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 69.3, "median_query_seconds": 55.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.3": { "accuracy": 70.047, "latency": 24.725, "stderr": 1.564, "cost_per_test": 0.081293, "avg_input_tokens": 60993.0, "avg_output_tokens": 2033.8, "avg_reasoning_tokens": 1583.2, "avg_cache_read_tokens": 31.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 24.7, "median_query_seconds": 20.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 69.814, "latency": 51.921, "stderr": 1.567, "cost_per_test": 0.021752, "avg_input_tokens": 63890.1, "avg_output_tokens": 2250.4, "avg_cache_read_tokens": 482.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 51.9, "median_query_seconds": 23.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 69.697, "latency": 23.548, "stderr": 1.569, "cost_per_test": 0.041574, "avg_input_tokens": 65533.1, "avg_output_tokens": 959.6, "avg_cache_read_tokens": 101.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 23.5, "median_query_seconds": 17.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.5": { "accuracy": 69.58, "latency": 26.764, "stderr": 1.571, "cost_per_test": 0.362691, "avg_input_tokens": 63882.1, "avg_output_tokens": 1444.3, "avg_reasoning_tokens": 1164.9, "avg_cache_read_tokens": 10.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 26.8, "median_query_seconds": 13.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-0709": { "accuracy": 69.347, "latency": 33.813, "stderr": 1.574, "cost_per_test": 0.204668, "avg_input_tokens": 60994.4, "avg_output_tokens": 1445.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 33.8, "median_query_seconds": 26.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 69.231, "latency": 45.083, "stderr": 1.576, "cost_per_test": 0.072205, "avg_input_tokens": 63890.1, "avg_output_tokens": 2053.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 45.1, "median_query_seconds": 22.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 68.998, "latency": 22.376, "stderr": 1.579, "cost_per_test": 0.478497, "avg_input_tokens": 72397.9, "avg_output_tokens": 1045.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 22.4, "median_query_seconds": 17.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 68.531, "latency": 88.951, "stderr": 1.585, "cost_per_test": 0.198727, "avg_input_tokens": 66087.0, "avg_output_tokens": 3352.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 89.0, "median_query_seconds": 79.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 68.415, "latency": 12.076, "stderr": 1.587, "cost_per_test": 0.131473, "avg_input_tokens": 62250.3, "avg_output_tokens": 1218.8, "avg_reasoning_tokens": 941.2, "avg_cache_read_tokens": 227.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 12.1, "median_query_seconds": 9.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 68.182, "latency": 96.974, "stderr": 1.59, "cost_per_test": 0.042064, "avg_input_tokens": 64717.5, "avg_output_tokens": 2006.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 34736.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 97.0, "median_query_seconds": 93.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 48000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 67.949, "latency": 20.457, "stderr": 1.593, "cost_per_test": 0.416108, "avg_input_tokens": 102472.5, "avg_output_tokens": 2134.5, "avg_cache_read_tokens": 52.5, "avg_cache_write_tokens": 102419.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.5, "median_query_seconds": 14.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-max-preview": { "accuracy": 67.949, "latency": 51.146, "stderr": 1.593, "cost_per_test": 0.102408, "avg_input_tokens": 65927.1, "avg_output_tokens": 1877.6, "avg_reasoning_tokens": 1540.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 51.1, "median_query_seconds": 45.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 67.832, "latency": 24.057, "stderr": 1.595, "cost_per_test": 0.132183, "avg_input_tokens": 62254.6, "avg_output_tokens": 1375.5, "avg_reasoning_tokens": 1140.4, "avg_cache_read_tokens": 386.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 24.1, "median_query_seconds": 17.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 67.832, "latency": 40.886, "stderr": 1.595, "cost_per_test": 0.034482, "avg_input_tokens": 65927.9, "avg_output_tokens": 3379.5, "turn_metrics": { "tasks_measured": 857, "avg_turns": 1.0, "avg_query_seconds": 40.9, "median_query_seconds": 36.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 67.716, "latency": 14.833, "stderr": 1.596, "cost_per_test": 0.039017, "avg_input_tokens": 66440.4, "avg_output_tokens": 1932.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 14.8, "median_query_seconds": 8.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 67.599, "latency": 18.561, "stderr": 1.598, "cost_per_test": 0.012735, "avg_input_tokens": 61143.3, "avg_output_tokens": 1084.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.6, "median_query_seconds": 14.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 67.366, "latency": 55.876, "stderr": 1.601, "cost_per_test": 0.146023, "avg_input_tokens": 65965.1, "avg_output_tokens": 2348.8, "avg_reasoning_tokens": 1979.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 55.9, "median_query_seconds": 32.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5-thinking": { "accuracy": 67.016, "latency": 70.517, "stderr": 1.605, "cost_per_test": 0.073316, "avg_input_tokens": 64805.8, "avg_output_tokens": 2689.6, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 70.5, "median_query_seconds": 56.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 66.783, "latency": 38.144, "stderr": 1.608, "cost_per_test": 0.002385, "avg_input_tokens": 63653.0, "avg_output_tokens": 2445.6, "avg_reasoning_tokens": 2238.3, "avg_cache_read_tokens": 63590.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 38.1, "median_query_seconds": 21.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 66.783, "latency": 122.651, "stderr": 1.608, "cost_per_test": 0.197126, "avg_input_tokens": 63882.1, "avg_output_tokens": 2494.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 122.7, "median_query_seconds": 48.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 66.667, "latency": 20.037, "stderr": 1.609, "cost_per_test": 0.287644, "avg_input_tokens": 72397.9, "avg_output_tokens": 1076.9, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.0, "median_query_seconds": 14.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 66.434, "latency": 9.294, "stderr": 1.612, "cost_per_test": 0.138028, "avg_input_tokens": 63882.1, "avg_output_tokens": 859.3, "avg_reasoning_tokens": 595.9, "avg_cache_read_tokens": 26.6, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 9.3, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 66.434, "latency": 20.315, "stderr": 1.612, "cost_per_test": 0.367275, "avg_input_tokens": 63887.7, "avg_output_tokens": 1597.1, "avg_reasoning_tokens": 1364.4, "avg_cache_read_tokens": 16.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.3, "median_query_seconds": 8.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 66.317, "latency": 26.348, "stderr": 1.614, "cost_per_test": 0.0982, "avg_input_tokens": 64811.3, "avg_output_tokens": 1696.4, "avg_reasoning_tokens": 1392.8, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 26.3, "median_query_seconds": 18.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 66.2, "latency": 20.361, "stderr": 1.615, "cost_per_test": 0.475505, "avg_input_tokens": 72409.9, "avg_output_tokens": 923.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.4, "median_query_seconds": 17.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 66.084, "latency": 18.221, "stderr": 1.616, "cost_per_test": 0.654716, "avg_input_tokens": 102478.5, "avg_output_tokens": 1163.9, "avg_cache_read_tokens": 2585.5, "avg_cache_write_tokens": 99888.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.2, "median_query_seconds": 14.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 65.967, "latency": 20.38, "stderr": 1.618, "cost_per_test": 0.679358, "avg_input_tokens": 102472.5, "avg_output_tokens": 1562.3, "avg_cache_read_tokens": 26.3, "avg_cache_write_tokens": 102445.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.4, "median_query_seconds": 16.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 65.851, "latency": 9.418, "stderr": 1.619, "cost_per_test": 0.130871, "avg_input_tokens": 61111.3, "avg_output_tokens": 1441.5, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 9.4, "median_query_seconds": 7.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 65.734, "latency": 28.137, "stderr": 1.62, "cost_per_test": 0.145587, "avg_input_tokens": 66440.4, "avg_output_tokens": 1058.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 28.1, "median_query_seconds": 21.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 65.734, "latency": 31.057, "stderr": 1.62, "cost_per_test": 0.109551, "avg_input_tokens": 63882.1, "avg_output_tokens": 2132.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 31.1, "median_query_seconds": 17.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 65.501, "latency": 10.865, "stderr": 1.623, "cost_per_test": 0.129954, "avg_input_tokens": 62566.2, "avg_output_tokens": 602.7, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 10.9, "median_query_seconds": 9.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 65.385, "latency": 18.336, "stderr": 1.624, "cost_per_test": 0.17462, "avg_input_tokens": 65927.1, "avg_output_tokens": 1306.9, "avg_reasoning_tokens": 1022.1, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.3, "median_query_seconds": 13.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.1": { "accuracy": 65.268, "latency": 45.736, "stderr": 1.625, "cost_per_test": 0.071771, "avg_input_tokens": 64808.5, "avg_output_tokens": 2335.1, "avg_reasoning_tokens": 2024.5, "avg_cache_read_tokens": 636.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 45.7, "median_query_seconds": 33.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 65.035, "latency": 20.103, "stderr": 1.628, "cost_per_test": 0.01626, "avg_input_tokens": 63882.1, "avg_output_tokens": 2907.8, "avg_reasoning_tokens": 2651.1, "avg_cache_read_tokens": 31.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.1, "median_query_seconds": 8.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 65.035, "latency": 32.888, "stderr": 1.628, "cost_per_test": 0.090475, "avg_input_tokens": 63882.1, "avg_output_tokens": 1433.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 32.9, "median_query_seconds": 23.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 64.918, "latency": 15.701, "stderr": 1.629, "cost_per_test": null, "avg_input_tokens": 70731.3, "avg_output_tokens": 1251.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 15.7, "median_query_seconds": 12.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 64.569, "latency": 8.593, "stderr": 1.633, "cost_per_test": 0.113913, "avg_input_tokens": 66440.4, "avg_output_tokens": 1583.7, "avg_reasoning_tokens": 1306.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 8.6, "median_query_seconds": 6.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 64.569, "latency": 23.778, "stderr": 1.633, "cost_per_test": 0.154515, "avg_input_tokens": 66442.3, "avg_output_tokens": 1825.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 23.8, "median_query_seconds": 18.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 64.452, "latency": 119.041, "stderr": 1.634, "cost_per_test": 0.232409, "avg_input_tokens": 72409.0, "avg_output_tokens": 1012.1, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 119.0, "median_query_seconds": 52.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 64.219, "latency": 21.997, "stderr": 1.636, "cost_per_test": 0.286695, "avg_input_tokens": 72409.9, "avg_output_tokens": 1011.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 22.0, "median_query_seconds": 18.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 64.219, "latency": 34.465, "stderr": 1.636, "cost_per_test": 0.007994, "avg_input_tokens": 65928.1, "avg_output_tokens": 3504.0, "avg_reasoning_tokens": 3169.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 34.5, "median_query_seconds": 30.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 64.219, "latency": 89.86, "stderr": 1.636, "cost_per_test": 0.041781, "avg_input_tokens": 64294.1, "avg_output_tokens": 1908.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 89.9, "median_query_seconds": 61.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 63.753, "latency": 11.623, "stderr": 1.641, "cost_per_test": 0.023635, "avg_input_tokens": 60928.7, "avg_output_tokens": 669.2, "avg_cache_read_tokens": 47.2, "avg_cache_write_tokens": 60881.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 11.6, "median_query_seconds": 11.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 63.52, "latency": 6.733, "stderr": 1.643, "cost_per_test": 0.110554, "avg_input_tokens": 66440.4, "avg_output_tokens": 1452.5, "avg_reasoning_tokens": 1193.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 6.7, "median_query_seconds": 5.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-3": { "accuracy": 63.52, "latency": 17.873, "stderr": 1.643, "cost_per_test": 0.184299, "avg_input_tokens": 59742.8, "avg_output_tokens": 338.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 17.9, "median_query_seconds": 14.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 63.17, "latency": 15.049, "stderr": 1.647, "cost_per_test": 0.096327, "avg_input_tokens": 72409.9, "avg_output_tokens": 1188.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 15.0, "median_query_seconds": 13.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.6": { "accuracy": 63.17, "latency": 62.917, "stderr": 1.647, "cost_per_test": 0.040881, "avg_input_tokens": 64841.8, "avg_output_tokens": 941.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 62.9, "median_query_seconds": 47.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-4.5": { "accuracy": 63.054, "latency": 92.297, "stderr": 1.648, "cost_per_test": 0.040809, "avg_input_tokens": 64659.1, "avg_output_tokens": 1001.9, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 92.3, "median_query_seconds": 79.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 62.587, "latency": 28.746, "stderr": 1.652, "cost_per_test": 0.081072, "avg_input_tokens": 62789.5, "avg_output_tokens": 258.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 28.7, "median_query_seconds": 26.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 62.471, "latency": 16.623, "stderr": 1.653, "cost_per_test": 0.029339, "avg_input_tokens": 66328.0, "avg_output_tokens": 732.0, "avg_reasoning_tokens": 394.6, "avg_cache_read_tokens": 348.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 16.6, "median_query_seconds": 15.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 62.354, "latency": 10.956, "stderr": 1.654, "cost_per_test": 0.431934, "avg_input_tokens": 72380.9, "avg_output_tokens": 266.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 11.0, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-27b": { "accuracy": 62.354, "latency": 51.777, "stderr": 1.654, "cost_per_test": 0.046263, "avg_input_tokens": 65931.5, "avg_output_tokens": 1862.1, "avg_reasoning_tokens": 1595.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 857, "avg_turns": 1.0, "avg_query_seconds": 51.8, "median_query_seconds": 48.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 62.354, "latency": 158.5, "stderr": 1.654, "cost_per_test": 0.079354, "avg_input_tokens": 61540.4, "avg_output_tokens": 242.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 158.5, "median_query_seconds": 144.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 62.238, "latency": 4.411, "stderr": 1.655, "cost_per_test": 0.023324, "avg_input_tokens": 66440.4, "avg_output_tokens": 1356.9, "avg_reasoning_tokens": 1132.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 4.4, "median_query_seconds": 3.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 62.238, "latency": 12.856, "stderr": 1.655, "cost_per_test": 0.005258, "avg_input_tokens": 67014.1, "avg_output_tokens": 1053.3, "avg_cache_read_tokens": 0.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 12.9, "median_query_seconds": 8.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 62.238, "latency": 23.407, "stderr": 1.655, "cost_per_test": 0.228911, "avg_input_tokens": 70930.1, "avg_output_tokens": 1074.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 23.4, "median_query_seconds": 22.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 18384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 62.238, "latency": 35.919, "stderr": 1.655, "cost_per_test": 0.038254, "avg_input_tokens": 65927.1, "avg_output_tokens": 1763.6, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 35.9, "median_query_seconds": 31.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 62.005, "latency": 17.433, "stderr": 1.657, "cost_per_test": 0.009434, "avg_input_tokens": 63640.0, "avg_output_tokens": 1872.4, "avg_reasoning_tokens": 1659.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 17.4, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 61.888, "latency": 30.263, "stderr": 1.658, "cost_per_test": 0.020614, "avg_input_tokens": 66468.1, "avg_output_tokens": 269.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 30.3, "median_query_seconds": 19.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 61.888, "latency": 74.027, "stderr": 1.658, "cost_per_test": 0.122195, "avg_input_tokens": 63640.0, "avg_output_tokens": 3293.4, "avg_reasoning_tokens": 3083.6, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 74.0, "median_query_seconds": 30.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-4-5-20251001": { "accuracy": 61.772, "latency": 8.526, "stderr": 1.659, "cost_per_test": 0.073944, "avg_input_tokens": 72380.9, "avg_output_tokens": 312.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 8.5, "median_query_seconds": 7.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-pro-exp-03-25": { "accuracy": 61.772, "latency": 20.235, "stderr": 1.659, "cost_per_test": 0.084215, "avg_input_tokens": 65118.1, "avg_output_tokens": 281.7, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.2, "median_query_seconds": 14.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 61.655, "latency": 14.71, "stderr": 1.66, "cost_per_test": 0.018712, "avg_input_tokens": 59743.8, "avg_output_tokens": 1578.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 14.7, "median_query_seconds": 12.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-4.7": { "accuracy": 61.189, "latency": 65.391, "stderr": 1.664, "cost_per_test": 0.045032, "avg_input_tokens": 64856.1, "avg_output_tokens": 2794.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 65.4, "median_query_seconds": 44.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 61.189, "latency": 109.941, "stderr": 1.664, "cost_per_test": 0.019795, "avg_input_tokens": 63889.8, "avg_output_tokens": 1923.2, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 109.9, "median_query_seconds": 32.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 61.072, "latency": 7.363, "stderr": 1.665, "cost_per_test": 0.012353, "avg_input_tokens": 63883.6, "avg_output_tokens": 620.1, "avg_reasoning_tokens": 362.1, "avg_cache_read_tokens": 6660.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 7.4, "median_query_seconds": 5.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 61.072, "latency": 22.18, "stderr": 1.665, "cost_per_test": 0.024085, "avg_input_tokens": 61021.5, "avg_output_tokens": 1015.4, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 22.2, "median_query_seconds": 17.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 60.839, "latency": 4.708, "stderr": 1.666, "cost_per_test": 0.018404, "avg_input_tokens": 66440.4, "avg_output_tokens": 1291.7, "avg_reasoning_tokens": 1068.3, "avg_cache_read_tokens": 639.5, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 4.7, "median_query_seconds": 4.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 60.839, "latency": 40.371, "stderr": 1.666, "cost_per_test": 0.010328, "avg_input_tokens": 63723.8, "avg_output_tokens": 1302.9, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 40.4, "median_query_seconds": 38.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929": { "accuracy": 60.49, "latency": 14.192, "stderr": 1.669, "cost_per_test": 0.22031, "avg_input_tokens": 72366.3, "avg_output_tokens": 300.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 14.2, "median_query_seconds": 12.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 60.373, "latency": 17.59, "stderr": 1.67, "cost_per_test": 0.025373, "avg_input_tokens": 60948.7, "avg_output_tokens": 1054.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 17.6, "median_query_seconds": 13.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 60.256, "latency": 28.917, "stderr": 1.671, "cost_per_test": 0.020583, "avg_input_tokens": 66468.1, "avg_output_tokens": 257.0, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 28.9, "median_query_seconds": 17.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 60.256, "latency": 45.279, "stderr": 1.671, "cost_per_test": 0.068348, "avg_input_tokens": 63886.5, "avg_output_tokens": 4540.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 45.3, "median_query_seconds": 27.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 60.14, "latency": 19.334, "stderr": 1.672, "cost_per_test": 0.008752, "avg_input_tokens": 66320.0, "avg_output_tokens": 623.2, "avg_reasoning_tokens": 290.0, "avg_cache_read_tokens": 5158.4, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 19.3, "median_query_seconds": 17.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 59.907, "latency": 42.14, "stderr": 1.673, "cost_per_test": 0.127154, "avg_input_tokens": 70731.3, "avg_output_tokens": 2807.6, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 42.1, "median_query_seconds": 35.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 59.557, "latency": 18.054, "stderr": 1.675, "cost_per_test": 0.131154, "avg_input_tokens": 62565.2, "avg_output_tokens": 753.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.1, "median_query_seconds": 15.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 59.557, "latency": 73.135, "stderr": 1.675, "cost_per_test": 0.035322, "avg_input_tokens": 70649.8, "avg_output_tokens": 474.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 73.1, "median_query_seconds": 52.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 59.324, "latency": 7.8, "stderr": 1.677, "cost_per_test": 0.025626, "avg_input_tokens": 62566.2, "avg_output_tokens": 374.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 7.8, "median_query_seconds": 7.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 59.324, "latency": 11.07, "stderr": 1.677, "cost_per_test": 0.018464, "avg_input_tokens": 59743.8, "avg_output_tokens": 1082.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 11.1, "median_query_seconds": 9.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 58.858, "latency": 35.154, "stderr": 1.68, "cost_per_test": 0.006747, "avg_input_tokens": 66468.1, "avg_output_tokens": 251.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 35.2, "median_query_seconds": 22.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 57.925, "latency": 17.095, "stderr": 1.685, "cost_per_test": 0.073049, "avg_input_tokens": 62565.2, "avg_output_tokens": 960.7, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 17.1, "median_query_seconds": 14.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 56.527, "latency": 146.262, "stderr": 1.692, "cost_per_test": null, "avg_input_tokens": 69030.4, "avg_output_tokens": 1078.2, "avg_reasoning_tokens": 709.2, "avg_cache_read_tokens": 32.3, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 146.3, "median_query_seconds": 139.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 56.294, "latency": 7.342, "stderr": 1.693, "cost_per_test": 0.042996, "avg_input_tokens": 61114.3, "avg_output_tokens": 61546.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 7.3, "median_query_seconds": 4.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 55.361, "latency": 5.088, "stderr": 1.697, "cost_per_test": 0.020182, "avg_input_tokens": 65090.9, "avg_output_tokens": 261.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 5.1, "median_query_seconds": 2.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max": { "accuracy": 55.361, "latency": 43.689, "stderr": 1.697, "cost_per_test": 0.080507, "avg_input_tokens": 66085.0, "avg_output_tokens": 200.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 43.7, "median_query_seconds": 36.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 55.245, "latency": 34.139, "stderr": 1.698, "cost_per_test": 0.006824, "avg_input_tokens": 66468.1, "avg_output_tokens": 442.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 34.1, "median_query_seconds": 22.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 55.012, "latency": 139.86, "stderr": 1.698, "cost_per_test": null, "avg_input_tokens": 69067.4, "avg_output_tokens": 999.0, "avg_reasoning_tokens": 602.3, "avg_cache_read_tokens": 47.3, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 139.9, "median_query_seconds": 140.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 54.895, "latency": 235.587, "stderr": 1.699, "cost_per_test": 0.080618, "avg_input_tokens": 66085.0, "avg_output_tokens": 219.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 235.6, "median_query_seconds": 192.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "arcee-ai/trinity-large-thinking": { "accuracy": 54.779, "latency": 11.372, "stderr": 1.699, "cost_per_test": 0.015928, "avg_input_tokens": 61199.6, "avg_output_tokens": 715.6, "avg_reasoning_tokens": 335.6, "avg_cache_read_tokens": 85.9, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 11.4, "median_query_seconds": 9.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Arcee-Ai", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 53.497, "latency": 43.407, "stderr": 1.703, "cost_per_test": 0.005289, "avg_input_tokens": 63734.3, "avg_output_tokens": 2759.2, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 43.4, "median_query_seconds": 39.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 52.914, "latency": 31.344, "stderr": 1.704, "cost_per_test": 0.056273, "avg_input_tokens": 62251.2, "avg_output_tokens": 274.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 31.3, "median_query_seconds": 25.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 52.914, "latency": 104.11, "stderr": 1.704, "cost_per_test": 0.221455, "avg_input_tokens": 72381.0, "avg_output_tokens": 287.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 104.1, "median_query_seconds": 48.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 52.564, "latency": 36.63, "stderr": 1.705, "cost_per_test": 0.088122, "avg_input_tokens": 62251.2, "avg_output_tokens": 756.1, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 36.6, "median_query_seconds": 32.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 52.331, "latency": 29.311, "stderr": 1.705, "cost_per_test": 0.004211, "avg_input_tokens": 70665.4, "avg_output_tokens": 3633.6, "avg_cache_read_tokens": 1214.8, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 29.3, "median_query_seconds": 15.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 51.282, "latency": 0.046, "stderr": 1.706, "cost_per_test": 0.058611, "avg_input_tokens": 72381.0, "avg_output_tokens": 176.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.0, "median_query_seconds": 0.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3": { "accuracy": 51.049, "latency": 27.684, "stderr": 1.707, "cost_per_test": 0.05626, "avg_input_tokens": 62251.2, "avg_output_tokens": 259.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 27.7, "median_query_seconds": 21.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-2-1212": { "accuracy": 50.932, "latency": 15.85, "stderr": 1.707, "cost_per_test": 0.129243, "avg_input_tokens": 63492.7, "avg_output_tokens": 225.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 15.8, "median_query_seconds": 10.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 49.767, "latency": 0.052, "stderr": 1.707, "cost_per_test": 0.219957, "avg_input_tokens": 72381.0, "avg_output_tokens": 187.6, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.1, "median_query_seconds": 0.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3p1": { "accuracy": 49.301, "latency": 60.891, "stderr": 1.707, "cost_per_test": 0.035907, "avg_input_tokens": 63524.1, "avg_output_tokens": 198.4, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 60.9, "median_query_seconds": 55.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 49.301, "latency": 109.73, "stderr": 1.707, "cost_per_test": 0.037839, "avg_input_tokens": 63005.2, "avg_output_tokens": 1548.0, "turn_metrics": { "tasks_measured": 847, "avg_turns": 1.0, "avg_query_seconds": 109.7, "median_query_seconds": 72.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 48.718, "latency": 42.666, "stderr": 1.706, "cost_per_test": 0.064891, "avg_input_tokens": 64559.7, "avg_output_tokens": 110.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 42.7, "median_query_seconds": 31.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 48.135, "latency": 4.369, "stderr": 1.706, "cost_per_test": 0.013864, "avg_input_tokens": 62034.1, "avg_output_tokens": 245.8, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 4.4, "median_query_seconds": 3.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 47.203, "latency": 51.99, "stderr": 1.704, "cost_per_test": 0.148302, "avg_input_tokens": 70041.7, "avg_output_tokens": 2064.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 52.0, "median_query_seconds": 39.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 46.503, "latency": 7.659, "stderr": 1.703, "cost_per_test": 0.012319, "avg_input_tokens": 61155.3, "avg_output_tokens": 230.4, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 7.7, "median_query_seconds": 6.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 46.154, "latency": 35.417, "stderr": 1.702, "cost_per_test": 0.028549, "avg_input_tokens": 70140.3, "avg_output_tokens": 246.5, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 35.4, "median_query_seconds": 23.5, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 45.921, "latency": 23.233, "stderr": 1.701, "cost_per_test": 0.073609, "avg_input_tokens": 62565.2, "avg_output_tokens": 1088.1, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 23.2, "median_query_seconds": 21.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 45.455, "latency": 40.394, "stderr": 1.7, "cost_per_test": 0.035821, "avg_input_tokens": 63535.1, "avg_output_tokens": 168.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 40.4, "median_query_seconds": 34.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 44.522, "latency": 6.467, "stderr": 1.697, "cost_per_test": 0.011399, "avg_input_tokens": 62034.1, "avg_output_tokens": 395.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 6.5, "median_query_seconds": 4.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 43.124, "latency": 0.056, "stderr": 1.691, "cost_per_test": 0.009254, "avg_input_tokens": 62458.8, "avg_output_tokens": 139.9, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.1, "median_query_seconds": 0.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 41.608, "latency": 12.926, "stderr": 1.683, "cost_per_test": 0.00524, "avg_input_tokens": 68715.7, "avg_output_tokens": 286.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 12.9, "median_query_seconds": 5.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.0-pro-exp-02-05": { "accuracy": 40.443, "latency": 15.506, "stderr": 1.675, "cost_per_test": 0.081991, "avg_input_tokens": 65118.1, "avg_output_tokens": 118.6, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 15.5, "median_query_seconds": 10.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 40.443, "latency": 20.551, "stderr": 1.675, "cost_per_test": 0.082015, "avg_input_tokens": 65117.1, "avg_output_tokens": 123.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 20.6, "median_query_seconds": 16.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 39.977, "latency": 14.674, "stderr": 1.672, "cost_per_test": 0.035917, "avg_input_tokens": 70130.7, "avg_output_tokens": 596.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 14.7, "median_query_seconds": 10.4, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 39.86, "latency": 6.248, "stderr": 1.672, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 39.744, "latency": 10.862, "stderr": 1.671, "cost_per_test": 0.166879, "avg_input_tokens": 65874.6, "avg_output_tokens": 219.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 10.9, "median_query_seconds": 10.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 39.044, "latency": 33.916, "stderr": 1.665, "cost_per_test": 0.139671, "avg_input_tokens": 69003.0, "avg_output_tokens": 208.1, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 33.9, "median_query_seconds": 10.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 38.811, "latency": 5.045, "stderr": 1.664, "cost_per_test": 0.006318, "avg_input_tokens": 62566.2, "avg_output_tokens": 152.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 5.0, "median_query_seconds": 4.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 37.879, "latency": 13.387, "stderr": 1.656, "cost_per_test": 0.004915, "avg_input_tokens": 65117.1, "avg_output_tokens": 103.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 13.4, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { "accuracy": 37.646, "latency": 0.068, "stderr": 1.654, "cost_per_test": 0.057026, "avg_input_tokens": 64676.3, "avg_output_tokens": 125.7, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.1, "median_query_seconds": 0.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 37.063, "latency": 3.391, "stderr": 1.649, "cost_per_test": 0.013854, "avg_input_tokens": 69003.0, "avg_output_tokens": 133.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 3.4, "median_query_seconds": 2.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "together/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { "accuracy": 36.946, "latency": 0.053, "stderr": 1.648, "cost_per_test": 0.011684, "avg_input_tokens": 64676.3, "avg_output_tokens": 237.3, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.1, "median_query_seconds": 0.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "ai21labs/jamba-1.5-large": { "accuracy": 36.48, "latency": 9.97, "stderr": 1.643, "cost_per_test": 0.138661, "avg_input_tokens": 69003.0, "avg_output_tokens": 81.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 10.0, "median_query_seconds": 9.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 35.664, "latency": 18.434, "stderr": 1.635, "cost_per_test": 0.006552, "avg_input_tokens": 65117.1, "avg_output_tokens": 99.5, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 18.4, "median_query_seconds": 9.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ai21labs/jamba-1.5-mini": { "accuracy": 31.935, "latency": 2.07, "stderr": 1.592, "cost_per_test": 0.013827, "avg_input_tokens": 69003.0, "avg_output_tokens": 65.2, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 2.1, "median_query_seconds": 2.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "openai/gpt-4o-2024-08-06": { "accuracy": 31.702, "latency": 0.009, "stderr": 1.589, "cost_per_test": 0.15394, "avg_input_tokens": 62458.8, "avg_output_tokens": 109.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.0, "median_query_seconds": 0.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-1.5-flash-001": { "accuracy": 21.445, "latency": 0.059, "stderr": 1.401, "cost_per_test": 0.005006, "avg_input_tokens": 66469.1, "avg_output_tokens": 67.9, "avg_reasoning_tokens": 0.0, "turn_metrics": { "tasks_measured": 858, "avg_turns": 1.0, "avg_query_seconds": 0.1, "median_query_seconds": 0.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } } } }