{ "metadata": { "benchmark": "IOI v1", "slug": "ioi-v1", "description": "Based on the International Olympiad in Informatics", "benchmark_id": "ioi_v1", "family": "ioi", "version": "1", "updated": "2026-08-09", "dataset_type": "public", "industry": "coding", "tasks": { "overall": "Overall", "ioi2024": "IOI 2024", "ioi2025": "IOI 2025" }, "models": [ "alibaba/qwen3-max", "alibaba/qwen3-max-2026-01-23", "alibaba/qwen3-max-preview", "alibaba/qwen3.7-max", "alibaba/qwen3.8-max", "anthropic/claude-fable-5", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-5", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929-thinking", "deepseek/deepseek-v4-pro", "fireworks/deepseek-v3-0324", "fireworks/deepseek-v3p2", "fireworks/deepseek-v3p2-thinking", "fireworks/qwen3-235b-a22b", "google/gemini-2.5-flash", "google/gemini-2.5-pro", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.5-flash-lite", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-code-fast-1", "kimi/kimi-k2.5-thinking", "meta/muse_spark_1_2", "minimax/MiniMax-M2.1", "minimax/MiniMax-M2.5", "minimax/MiniMax-M2.7", "mistralai/devstral-2512", "mistralai/labs-devstral-small-2512", "mistralai/magistral-medium-2509", "mistralai/mistral-large-2512", "openai/gpt-5-2025-08-07", "openai/gpt-5-codex", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.1-codex", "openai/gpt-5.1-codex-max", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.3-codex", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/o4-mini-2025-04-16", "together/moonshotai/Kimi-K2-Instruct", "zai/glm-4.5", "zai/glm-4.6", "zai/glm-4.7", "zai/glm-5-thinking" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "custom", "mode": "agentic", "archived": true, "partner": false, "total_models": 62 }, "tasks": { "overall": { "anthropic/claude-opus-5": { "accuracy": 91.667, "latency": 1693.04, "stderr": 8.333, "cost_per_test": 5.541014, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 86.667, "latency": 1310.679, "stderr": 1.667, "cost_per_test": 2.98545, "token_totals": { "input_tokens": 8111299, "output_tokens": 362342, "reasoning_tokens": 307844, "cache_read_tokens": 7447568, "cache_write_tokens": null }, "token_tasks_measured": 6, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 73.0, "latency": 3946.139, "stderr": 1.0, "cost_per_test": 2.392116, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 72.917, "latency": 1670.558, "stderr": 5.083, "cost_per_test": 0.262867, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 72.25, "latency": 3318.969, "stderr": 9.357, "cost_per_test": 15.101678, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 67.834, "latency": 2660.886, "stderr": 9.867, "cost_per_test": 3.471332, "avg_input_tokens": 3111734.8, "avg_output_tokens": 127091.0, "avg_reasoning_tokens": 91293.3, "avg_cache_read_tokens": 2761941.4, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 65.25, "latency": 1330.749, "stderr": 0.917, "cost_per_test": 0.97448, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 54.833, "latency": 6739.612, "stderr": 8.318, "cost_per_test": 3.871992, "avg_input_tokens": 2146870.9, "avg_output_tokens": 142786.0, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 49.5, "latency": 4771.52, "stderr": 3.667, "cost_per_test": 6.100817, "token_totals": { "input_tokens": 91784656, "output_tokens": 2070730, "reasoning_tokens": 1453110, "cache_read_tokens": 79024104, "cache_write_tokens": 0 }, "token_tasks_measured": 6, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 47.084, "latency": 3081.905, "stderr": 8.704, "cost_per_test": 10.486954, "avg_input_tokens": 3958856.4, "avg_output_tokens": 227220.5, "avg_cache_read_tokens": 3467196.1, "avg_cache_write_tokens": 491632.8, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 46.75, "latency": 4380.294, "stderr": 10.263, "cost_per_test": 4.859648, "avg_input_tokens": 2893590.0, "avg_output_tokens": 287807.0, "avg_reasoning_tokens": 264261.0, "avg_cache_read_tokens": 2014613.3, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 43.834, "latency": 3574.257, "stderr": 7.81, "cost_per_test": 2.020025, "avg_input_tokens": 2128679.8, "avg_output_tokens": 96712.9, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 39.084, "latency": 1846.526, "stderr": 10.545, "cost_per_test": 0.375052, "avg_input_tokens": 1286051.5, "avg_output_tokens": 70681.2, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 38.834, "latency": 900.44, "stderr": 11.492, "cost_per_test": 0.331188, "avg_input_tokens": 104174.5, "avg_output_tokens": 10236.6, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 35.833, "latency": 5373.876, "stderr": 7.139, "cost_per_test": 0.568564, "avg_input_tokens": 6758593.9, "avg_output_tokens": 189617.6, "avg_reasoning_tokens": 134333.0, "avg_cache_read_tokens": 6542402.2, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 30.166, "latency": 896.93, "stderr": 7.489, "cost_per_test": 0.709499, "avg_input_tokens": 610400.0, "avg_output_tokens": 75423.9, "avg_reasoning_tokens": 70471.4, "avg_cache_read_tokens": 556835.2, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 26.167, "latency": 882.37, "stderr": 1.167, "cost_per_test": 0.636385, "token_totals": { "input_tokens": 24147621, "output_tokens": 423418, "reasoning_tokens": 222934, "cache_read_tokens": 16609343, "cache_write_tokens": 0 }, "token_tasks_measured": 6, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-0709": { "accuracy": 26.167, "latency": 4265.882, "stderr": 6.763, "cost_per_test": 2.693986, "avg_input_tokens": 630057.2, "avg_output_tokens": 86144.8, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 23.584, "latency": 7682.383, "stderr": 7.547, "cost_per_test": 3.048735, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "zai/glm-5-thinking": { "accuracy": 22.0, "latency": 6806.938, "stderr": 7.832, "cost_per_test": 3.273674, "avg_input_tokens": 4847320.8, "avg_output_tokens": 112527.6, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 21.5, "latency": 1750.069, "stderr": 7.336, "cost_per_test": 0.368193, "avg_input_tokens": 183697.8, "avg_output_tokens": 13857.1, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-codex-max": { "accuracy": 21.416, "latency": 956.692, "stderr": 6.995, "cost_per_test": 0.422645, "avg_input_tokens": 256701.2, "avg_output_tokens": 10176.8, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 20.25, "latency": 8211.492, "stderr": 5.157, "cost_per_test": 3.9235, "avg_input_tokens": 370.1, "avg_output_tokens": 156866.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 20.0, "latency": 2608.252, "stderr": 4.581, "cost_per_test": 1.629264, "avg_input_tokens": 275834.5, "avg_output_tokens": 128447.0, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 18.334, "latency": 3315.166, "stderr": 5.918, "cost_per_test": 3.07299, "avg_input_tokens": 802193.2, "avg_output_tokens": 44427.3, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 17.667, "latency": 3828.386, "stderr": 6.084, "cost_per_test": 0.677261, "avg_input_tokens": 2229026.1, "avg_output_tokens": 134792.8, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 17.084, "latency": 2964.799, "stderr": 6.17, "cost_per_test": 0.615461, "avg_input_tokens": 279613.5, "avg_output_tokens": 26594.4, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max": { "accuracy": 15.666, "latency": 2045.247, "stderr": 6.352, "cost_per_test": 0.824523, "avg_input_tokens": 520216.8, "avg_output_tokens": 33377.1, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.3": { "accuracy": 15.334, "latency": 628.816, "stderr": 6.313, "cost_per_test": 0.351754, "avg_input_tokens": 564315.1, "avg_output_tokens": 64106.7, "avg_reasoning_tokens": 57861.0, "avg_cache_read_tokens": 489434.7, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 15.25, "latency": 1170.492, "stderr": 6.438, "cost_per_test": 0.042833, "avg_input_tokens": 877443.2, "avg_output_tokens": 86687.1, "avg_reasoning_tokens": 73259.4, "avg_cache_read_tokens": 793642.6, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 14.416, "latency": 2384.516, "stderr": 4.496, "cost_per_test": 0.05196, "avg_input_tokens": 4249.6, "avg_output_tokens": 29512.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 13.75, "latency": 2953.233, "stderr": 3.547, "cost_per_test": 1.901446, "avg_input_tokens": 510043.0, "avg_output_tokens": 93580.4, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 12.516, "latency": 6124.896, "stderr": 4.913, "cost_per_test": 17.585952, "avg_input_tokens": 997542.3, "avg_output_tokens": 34970.9, "avg_reasoning_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 11.5, "latency": 1408.454, "stderr": 4.388, "cost_per_test": 0.97412, "avg_input_tokens": 1335858.5, "avg_output_tokens": 1413897.2, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 10.667, "latency": 1558.619, "stderr": 3.089, "cost_per_test": 0.058713, "avg_input_tokens": 50676.9, "avg_output_tokens": 18055.8, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5-codex": { "accuracy": 9.75, "latency": 1542.894, "stderr": 7.717, "cost_per_test": 2.669955, "avg_input_tokens": 1435225.2, "avg_output_tokens": 87592.4, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 7.75, "latency": 3372.048, "stderr": 3.165, "cost_per_test": 1.399136, "avg_input_tokens": 1277152.2, "avg_output_tokens": 37429.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 7.667, "latency": 7675.292, "stderr": 3.198, "cost_per_test": 0.06955, "avg_input_tokens": 270945.7, "avg_output_tokens": 30719.8, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-4.7": { "accuracy": 7.584, "latency": 5316.876, "stderr": 3.404, "cost_per_test": 0.443515, "avg_input_tokens": 1552932.6, "avg_output_tokens": 48322.6, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 6.75, "latency": 390.378, "stderr": 3.316, "cost_per_test": 0.039168, "avg_input_tokens": 69879.0, "avg_output_tokens": 10848.9, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "high", "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 6.666, "latency": 22887.998, "stderr": 2.641, "cost_per_test": 1.501416, "avg_input_tokens": 4332998.5, "avg_output_tokens": 86408.9, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 6.5, "latency": 3253.12, "stderr": 2.096, "cost_per_test": 2.388928, "avg_input_tokens": 679864.8, "avg_output_tokens": 23289.0, "avg_reasoning_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 6.417, "latency": 1896.804, "stderr": 3.607, "cost_per_test": 0.35057, "avg_input_tokens": 1265346.0, "avg_output_tokens": 132676.8, "avg_reasoning_tokens": 121166.0, "avg_cache_read_tokens": 1027200.0, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 6.166, "latency": 7415.352, "stderr": 2.515, "cost_per_test": 2.133406, "avg_input_tokens": 1786445.0, "avg_output_tokens": 69392.2, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 4.916, "latency": 3242.758, "stderr": 2.379, "cost_per_test": 1.154876, "avg_input_tokens": 3424865.0, "avg_output_tokens": 109783.1, "avg_cache_read_tokens": 16980.4, "avg_cache_write_tokens": 3490.8, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 4.834, "latency": 480.192, "stderr": 2.355, "cost_per_test": 0.156077, "avg_input_tokens": 44475.2, "avg_output_tokens": 24353.2, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 4.584, "latency": 2216.287, "stderr": 1.139, "cost_per_test": 1.329832, "avg_input_tokens": 361606.8, "avg_output_tokens": 16334.1, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.6": { "accuracy": 4.334, "latency": 7105.854, "stderr": 2.485, "cost_per_test": 1.516537, "avg_input_tokens": 2707515.0, "avg_output_tokens": 65549.0, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-code-fast-1": { "accuracy": 4.333, "latency": 1083.862, "stderr": 1.157, "cost_per_test": 1.61801, "avg_input_tokens": 873912.0, "avg_output_tokens": 962152.0, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 4.0, "latency": 6788.576, "stderr": 1.46, "cost_per_test": 3.073948, "avg_input_tokens": 5730229.8, "avg_output_tokens": 139221.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 3.834, "latency": 577.632, "stderr": 2.848, "cost_per_test": 0.12675, "avg_input_tokens": 172617.8, "avg_output_tokens": 184453.6, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-codex": { "accuracy": 3.666, "latency": 2227.442, "stderr": 3.347, "cost_per_test": 0.176857, "avg_input_tokens": 44275.2, "avg_output_tokens": 12151.2, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 3.084, "latency": 1533.288, "stderr": 1.749, "cost_per_test": 0.02997, "avg_input_tokens": 124370.1, "avg_output_tokens": 10190.6, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-4.5": { "accuracy": 2.917, "latency": 2336.43, "stderr": 0.987, "cost_per_test": 0.942275, "avg_input_tokens": 1292285.2, "avg_output_tokens": 75865.2, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-2.5-flash": { "accuracy": 2.611, "latency": 350.186, "stderr": 1.03, "cost_per_test": 0.036998, "avg_input_tokens": 58583.2, "avg_output_tokens": 7769.1, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/labs-devstral-small-2512": { "accuracy": 2.5, "latency": 7466.603, "stderr": 1.781, "cost_per_test": 0.50569, "avg_input_tokens": 4792353.6, "avg_output_tokens": 88182.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 2.334, "latency": 5210.402, "stderr": 1.3, "cost_per_test": 0.319345, "avg_input_tokens": 1611250.4, "avg_output_tokens": 82905.4, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 1.667, "latency": 171.479, "stderr": 0.9, "cost_per_test": 0.606484, "avg_input_tokens": 665967.5, "avg_output_tokens": 7903.6, "avg_reasoning_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 1.25, "latency": 1551.101, "stderr": 0.632, "cost_per_test": 2.367573, "avg_input_tokens": 2257202.1, "avg_output_tokens": 36790.4, "avg_reasoning_tokens": 0.0, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 0.666, "latency": 2211.783, "stderr": 0.444, "cost_per_test": 6.344222, "avg_input_tokens": 2965557.2, "avg_output_tokens": 82621.5, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/devstral-2512": { "accuracy": 0.666, "latency": 8044.645, "stderr": 0.608, "cost_per_test": 1.753753, "avg_input_tokens": 4070057.1, "avg_output_tokens": 62865.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 0.0, "latency": 352.565, "stderr": 0.0, "cost_per_test": 0.109574, "avg_input_tokens": 375794.5, "avg_output_tokens": 30567.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "ioi2024": { "anthropic/claude-opus-5": { "accuracy": 100.0, "latency": 1737.255, "stderr": 0.0, "cost_per_test": 6.45987, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 100.0, "latency": 2540.44, "stderr": 0.0, "cost_per_test": 15.68634, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 85.0, "latency": 1314.969, "stderr": 13.693, "cost_per_test": 3.002041, "token_totals": { "input_tokens": 10317678, "output_tokens": 349355, "reasoning_tokens": 282948, "cache_read_tokens": 9790399, "cache_write_tokens": null }, "token_tasks_measured": 6, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 78.0, "latency": 1730.527, "stderr": 13.904, "cost_per_test": 0.260799, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 72.0, "latency": 3819.907, "stderr": 13.392, "cost_per_test": 1.999003, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 67.167, "latency": 2241.026, "stderr": 14.878, "cost_per_test": 3.371664, "avg_input_tokens": 3085697.7, "avg_output_tokens": 124750.7, "avg_reasoning_tokens": 91532.3, "avg_cache_read_tokens": 2761706.7, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 66.167, "latency": 678.342, "stderr": 14.997, "cost_per_test": 0.818561, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 58.167, "latency": 2631.97, "stderr": 14.481, "cost_per_test": 1.818037, "avg_input_tokens": 2060938.7, "avg_output_tokens": 89108.8, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 53.167, "latency": 2569.514, "stderr": 14.854, "cost_per_test": 9.010059, "avg_input_tokens": 2313633.7, "avg_output_tokens": 196826.8, "avg_cache_read_tokens": 1803617.0, "avg_cache_write_tokens": 509996.8, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 45.833, "latency": 5252.146, "stderr": 15.842, "cost_per_test": 7.024904, "token_totals": { "input_tokens": 83449414, "output_tokens": 2270344, "reasoning_tokens": 1692829, "cache_read_tokens": 65283007, "cache_write_tokens": 0 }, "token_tasks_measured": 6, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 43.833, "latency": 7378.658, "stderr": 12.379, "cost_per_test": 4.02342, "avg_input_tokens": 2379847.3, "avg_output_tokens": 145795.8, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 41.0, "latency": 2307.779, "stderr": 17.098, "cost_per_test": 0.401804, "avg_input_tokens": 1552270.2, "avg_output_tokens": 72440.0, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 40.5, "latency": 4033.028, "stderr": 17.613, "cost_per_test": 4.69984, "avg_input_tokens": 2365448.3, "avg_output_tokens": 279020.2, "avg_reasoning_tokens": 261596.7, "avg_cache_read_tokens": 1469525.3, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 38.5, "latency": 1079.286, "stderr": 17.904, "cost_per_test": 0.328592, "avg_input_tokens": 105008.8, "avg_output_tokens": 9881.2, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 31.333, "latency": 5262.744, "stderr": 10.426, "cost_per_test": 0.580726, "avg_input_tokens": 7402429.6, "avg_output_tokens": 188545.0, "avg_reasoning_tokens": 131255.6, "avg_cache_read_tokens": 7183257.6, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 25.0, "latency": 1087.865, "stderr": 14.652, "cost_per_test": 0.708425, "token_totals": { "input_tokens": 27132786, "output_tokens": 473906, "reasoning_tokens": 240717, "cache_read_tokens": 18792784, "cache_write_tokens": 0 }, "token_tasks_measured": 6, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5-thinking": { "accuracy": 23.667, "latency": 7231.432, "stderr": 11.716, "cost_per_test": 2.864289, "avg_input_tokens": 4774476.5, "avg_output_tokens": 103191.8, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-0709": { "accuracy": 23.167, "latency": 4824.866, "stderr": 11.985, "cost_per_test": 1.953434, "avg_input_tokens": 548064.2, "avg_output_tokens": 85730.5, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 22.5, "latency": 622.262, "stderr": 12.057, "cost_per_test": 0.654763, "avg_input_tokens": 428960.3, "avg_output_tokens": 75398.0, "avg_reasoning_tokens": 71344.3, "avg_cache_read_tokens": 364192.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 21.5, "latency": 1069.464, "stderr": 12.27, "cost_per_test": 0.415113, "avg_input_tokens": 210142.2, "avg_output_tokens": 27157.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 20.667, "latency": 11576.195, "stderr": 11.43, "cost_per_test": 2.959818, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 19.0, "latency": 2159.535, "stderr": 10.263, "cost_per_test": 0.842034, "avg_input_tokens": 413361.8, "avg_output_tokens": 32533.2, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 19.0, "latency": 2583.917, "stderr": 10.263, "cost_per_test": 2.711026, "avg_input_tokens": 710801.0, "avg_output_tokens": 38574.8, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 18.667, "latency": 5926.555, "stderr": 9.657, "cost_per_test": 15.554335, "avg_input_tokens": 857014.0, "avg_output_tokens": 35988.3, "avg_reasoning_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 18.0, "latency": 4779.544, "stderr": 5.452, "cost_per_test": 3.923563, "avg_input_tokens": 331.0, "avg_output_tokens": 156876.3, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-5.1-codex-max": { "accuracy": 14.5, "latency": 975.205, "stderr": 9.757, "cost_per_test": 0.49262, "avg_input_tokens": 282354.3, "avg_output_tokens": 13967.7, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 14.167, "latency": 967.635, "stderr": 6.269, "cost_per_test": 0.035235, "avg_input_tokens": 461999.8, "avg_output_tokens": 78731.0, "avg_reasoning_tokens": 70120.7, "avg_cache_read_tokens": 430165.3, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 14.0, "latency": 1615.658, "stderr": 6.364, "cost_per_test": 0.052688, "avg_input_tokens": 4429.5, "avg_output_tokens": 29885.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.3": { "accuracy": 13.0, "latency": 555.026, "stderr": 6.272, "cost_per_test": 0.32626, "avg_input_tokens": 505701.7, "avg_output_tokens": 60952.2, "avg_reasoning_tokens": 55836.3, "avg_cache_read_tokens": 436426.7, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 12.333, "latency": 3278.996, "stderr": 5.124, "cost_per_test": 1.949747, "avg_input_tokens": 540495.8, "avg_output_tokens": 95207.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 12.167, "latency": 3195.697, "stderr": 5.041, "cost_per_test": 0.561257, "avg_input_tokens": 1353032.2, "avg_output_tokens": 127550.8, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 11.833, "latency": 1702.974, "stderr": 6.126, "cost_per_test": 0.363369, "avg_input_tokens": 168091.0, "avg_output_tokens": 15325.5, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 11.667, "latency": 1362.606, "stderr": 5.728, "cost_per_test": 2.276975, "avg_input_tokens": 2025083.6, "avg_output_tokens": 50378.2, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 11.167, "latency": 1374.876, "stderr": 5.32, "cost_per_test": 0.050024, "avg_input_tokens": 43612.3, "avg_output_tokens": 15238.7, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 11.0, "latency": 2627.572, "stderr": 4.967, "cost_per_test": 1.870345, "avg_input_tokens": 412865.0, "avg_output_tokens": 135426.3, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.7": { "accuracy": 10.5, "latency": 3615.545, "stderr": 5.882, "cost_per_test": 0.384617, "avg_input_tokens": 1258372.3, "avg_output_tokens": 47854.5, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 9.667, "latency": 2018.192, "stderr": 3.949, "cost_per_test": 2.080064, "avg_input_tokens": 567886.2, "avg_output_tokens": 25093.7, "avg_reasoning_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 9.333, "latency": 1370.235, "stderr": 4.305, "cost_per_test": 1.353118, "avg_input_tokens": 1872571.3, "avg_output_tokens": 1957208.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 7.667, "latency": 376.345, "stderr": 4.647, "cost_per_test": 0.114606, "avg_input_tokens": 31078.8, "avg_output_tokens": 18277.0, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 7.667, "latency": 4647.63, "stderr": 5.13, "cost_per_test": 0.047983, "avg_input_tokens": 175099.7, "avg_output_tokens": 25925.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 7.5, "latency": 17357.397, "stderr": 4.458, "cost_per_test": 1.508215, "avg_input_tokens": 4326244.0, "avg_output_tokens": 90341.0, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 6.667, "latency": 1876.864, "stderr": 4.513, "cost_per_test": 0.384694, "avg_input_tokens": 1638391.3, "avg_output_tokens": 137586.5, "avg_reasoning_tokens": 124562.8, "avg_cache_read_tokens": 1333674.7, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 6.5, "latency": 272.805, "stderr": 4.37, "cost_per_test": 0.033967, "avg_input_tokens": 58767.2, "avg_output_tokens": 9637.5, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "high", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 6.5, "latency": 4391.967, "stderr": 4.383, "cost_per_test": 2.10517, "avg_input_tokens": 1763331.0, "avg_output_tokens": 68367.8, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 6.5, "latency": 7790.385, "stderr": 2.786, "cost_per_test": 3.946165, "avg_input_tokens": 7374782.3, "avg_output_tokens": 172515.8, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 6.167, "latency": 307.059, "stderr": 5.629, "cost_per_test": 0.173256, "avg_input_tokens": 235702.7, "avg_output_tokens": 252230.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash": { "accuracy": 5.222, "latency": 623.818, "stderr": 2.059, "cost_per_test": 0.07068, "avg_input_tokens": 106115.4, "avg_output_tokens": 15538.2, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 5.167, "latency": 1431.232, "stderr": 1.892, "cost_per_test": 1.023628, "avg_input_tokens": 260701.8, "avg_output_tokens": 16101.5, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/labs-devstral-small-2512": { "accuracy": 3.667, "latency": 858.154, "stderr": 3.347, "cost_per_test": 0.513867, "avg_input_tokens": 4863815.8, "avg_output_tokens": 91619.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-code-fast-1": { "accuracy": 3.333, "latency": 867.087, "stderr": 1.925, "cost_per_test": 2.032726, "avg_input_tokens": 1110286.8, "avg_output_tokens": 1207112.7, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-4.5": { "accuracy": 3.167, "latency": 2172.987, "stderr": 1.535, "cost_per_test": 1.093628, "avg_input_tokens": 1532796.2, "avg_output_tokens": 79068.2, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5-codex": { "accuracy": 2.833, "latency": 1650.37, "stderr": 2.586, "cost_per_test": 2.583712, "avg_input_tokens": 1338925.5, "avg_output_tokens": 91005.5, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 1.667, "latency": 144.646, "stderr": 1.521, "cost_per_test": 0.106655, "avg_input_tokens": 113991.2, "avg_output_tokens": 4514.8, "avg_reasoning_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-4.6": { "accuracy": 1.0, "latency": 6110.323, "stderr": 0.577, "cost_per_test": 2.024354, "avg_input_tokens": 3101403.2, "avg_output_tokens": 74323.7, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 0.5, "latency": 1364.023, "stderr": 0.456, "cost_per_test": 0.01707, "avg_input_tokens": 70433.2, "avg_output_tokens": 5965.8, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 0.5, "latency": 1542.884, "stderr": 0.456, "cost_per_test": 2.42593, "avg_input_tokens": 2309624.7, "avg_output_tokens": 38768.5, "avg_reasoning_tokens": 0.0, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 0.5, "latency": 2735.47, "stderr": 0.456, "cost_per_test": 8.012136, "avg_input_tokens": 3750936.2, "avg_output_tokens": 102052.7, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.5, "latency": 3472.218, "stderr": 0.456, "cost_per_test": 1.262533, "avg_input_tokens": 3754837.0, "avg_output_tokens": 116984.5, "avg_cache_read_tokens": 17174.5, "avg_cache_write_tokens": 4495.5, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 0.0, "latency": 303.027, "stderr": 0.0, "cost_per_test": 0.066363, "avg_input_tokens": 194145.8, "avg_output_tokens": 26875.5, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/devstral-2512": { "accuracy": 0.0, "latency": 683.82, "stderr": 0.0, "cost_per_test": 1.817543, "avg_input_tokens": 4179678.7, "avg_output_tokens": 72836.0, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.1-codex": { "accuracy": 0.0, "latency": 2326.487, "stderr": 0.0, "cost_per_test": 0.201958, "avg_input_tokens": 45114.0, "avg_output_tokens": 14556.5, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 0.0, "latency": 4937.493, "stderr": 0.0, "cost_per_test": 0.233531, "avg_input_tokens": 824445.5, "avg_output_tokens": 86656.2, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null } }, "ioi2025": { "openai/gpt-5.6-sol": { "accuracy": 88.333, "latency": 1306.389, "stderr": 10.65, "cost_per_test": 2.96886, "token_totals": { "input_tokens": 5904920, "output_tokens": 375329, "reasoning_tokens": 332739, "cache_read_tokens": 5104736, "cache_write_tokens": null }, "token_tasks_measured": 6, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 83.333, "latency": 1648.826, "stderr": 16.667, "cost_per_test": 4.438387, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 74.0, "latency": 4072.372, "stderr": 15.882, "cost_per_test": 2.785229, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 68.5, "latency": 3080.746, "stderr": 12.965, "cost_per_test": 3.571001, "avg_input_tokens": 3137771.8, "avg_output_tokens": 129431.2, "avg_reasoning_tokens": 91054.3, "avg_cache_read_tokens": 2762176.0, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 67.833, "latency": 1610.589, "stderr": 15.511, "cost_per_test": 0.264936, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 65.833, "latency": 6100.567, "stderr": 11.114, "cost_per_test": 3.720564, "avg_input_tokens": 1913894.5, "avg_output_tokens": 139776.2, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 64.333, "latency": 1983.157, "stderr": 12.199, "cost_per_test": 1.130399, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 53.167, "latency": 4290.894, "stderr": 14.321, "cost_per_test": 5.176731, "token_totals": { "input_tokens": 100119897, "output_tokens": 1871115, "reasoning_tokens": 1213390, "cache_read_tokens": 92765202, "cache_write_tokens": 0 }, "token_tasks_measured": 6, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 53.0, "latency": 4727.561, "stderr": 10.541, "cost_per_test": 5.019455, "avg_input_tokens": 3421731.8, "avg_output_tokens": 296593.8, "avg_reasoning_tokens": 266925.3, "avg_cache_read_tokens": 2559701.3, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 44.5, "latency": 4097.498, "stderr": 18.714, "cost_per_test": 14.517016, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 41.0, "latency": 3594.296, "stderr": 9.077, "cost_per_test": 11.963848, "avg_input_tokens": 5604079.2, "avg_output_tokens": 257614.2, "avg_cache_read_tokens": 5130775.2, "avg_cache_write_tokens": 473268.8, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 40.333, "latency": 5485.007, "stderr": 9.754, "cost_per_test": 0.556402, "avg_input_tokens": 6114758.2, "avg_output_tokens": 190690.3, "avg_reasoning_tokens": 137410.3, "avg_cache_read_tokens": 5901546.7, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 39.167, "latency": 721.594, "stderr": 14.411, "cost_per_test": 0.333784, "avg_input_tokens": 103340.2, "avg_output_tokens": 10592.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 37.833, "latency": 1171.599, "stderr": 8.886, "cost_per_test": 0.764235, "avg_input_tokens": 791839.6, "avg_output_tokens": 75449.8, "avg_reasoning_tokens": 69598.4, "avg_cache_read_tokens": 749478.4, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 37.167, "latency": 1385.274, "stderr": 12.346, "cost_per_test": 0.3483, "avg_input_tokens": 1019832.8, "avg_output_tokens": 68922.3, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 31.167, "latency": 1797.164, "stderr": 13.332, "cost_per_test": 0.373017, "avg_input_tokens": 199304.5, "avg_output_tokens": 12388.7, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 29.5, "latency": 4516.544, "stderr": 5.854, "cost_per_test": 2.222013, "avg_input_tokens": 2196421.0, "avg_output_tokens": 104317.0, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-0709": { "accuracy": 29.167, "latency": 3706.897, "stderr": 6.269, "cost_per_test": 3.434538, "avg_input_tokens": 712050.2, "avg_output_tokens": 86559.2, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 29.0, "latency": 2588.931, "stderr": 7.699, "cost_per_test": 1.388182, "avg_input_tokens": 138804.0, "avg_output_tokens": 121467.7, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-codex-max": { "accuracy": 28.333, "latency": 938.178, "stderr": 10.027, "cost_per_test": 0.35267, "avg_input_tokens": 231048.2, "avg_output_tokens": 6386.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 27.333, "latency": 676.876, "stderr": 10.257, "cost_per_test": 0.564344, "token_totals": { "input_tokens": 21162456, "output_tokens": 372929, "reasoning_tokens": 205150, "cache_read_tokens": 14425902, "cache_write_tokens": 0 }, "token_tasks_measured": 6, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 26.5, "latency": 3788.571, "stderr": 9.857, "cost_per_test": 3.137652, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 23.167, "latency": 4461.074, "stderr": 11.074, "cost_per_test": 0.793265, "avg_input_tokens": 3105020.0, "avg_output_tokens": 142034.7, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 22.5, "latency": 11643.441, "stderr": 8.755, "cost_per_test": 3.923438, "avg_input_tokens": 409.2, "avg_output_tokens": 156855.7, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "zai/glm-5-thinking": { "accuracy": 20.333, "latency": 6382.444, "stderr": 10.397, "cost_per_test": 3.68306, "avg_input_tokens": 4920165.0, "avg_output_tokens": 121863.3, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.3": { "accuracy": 17.667, "latency": 702.606, "stderr": 10.959, "cost_per_test": 0.377249, "avg_input_tokens": 622928.5, "avg_output_tokens": 67261.2, "avg_reasoning_tokens": 59885.8, "avg_cache_read_tokens": 542442.7, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 17.667, "latency": 4046.414, "stderr": 5.896, "cost_per_test": 3.434954, "avg_input_tokens": 893585.5, "avg_output_tokens": 50279.8, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-codex": { "accuracy": 16.667, "latency": 1435.417, "stderr": 15.215, "cost_per_test": 2.756198, "avg_input_tokens": 1531525.0, "avg_output_tokens": 84179.2, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 16.333, "latency": 1373.348, "stderr": 11.246, "cost_per_test": 0.050431, "avg_input_tokens": 1292886.7, "avg_output_tokens": 94643.2, "avg_reasoning_tokens": 76398.2, "avg_cache_read_tokens": 1157120.0, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 15.167, "latency": 2627.47, "stderr": 4.907, "cost_per_test": 1.853145, "avg_input_tokens": 479590.2, "avg_output_tokens": 91953.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 15.167, "latency": 3770.063, "stderr": 6.853, "cost_per_test": 0.388888, "avg_input_tokens": 145865.2, "avg_output_tokens": 20655.7, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 14.833, "latency": 3153.373, "stderr": 6.353, "cost_per_test": 0.051233, "avg_input_tokens": 4069.8, "avg_output_tokens": 29139.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 13.667, "latency": 1446.674, "stderr": 7.647, "cost_per_test": 0.595122, "avg_input_tokens": 799145.7, "avg_output_tokens": 870586.3, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 10.167, "latency": 1742.362, "stderr": 3.14, "cost_per_test": 0.067402, "avg_input_tokens": 57741.5, "avg_output_tokens": 20873.0, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 9.833, "latency": 3021.03, "stderr": 3.295, "cost_per_test": 1.233933, "avg_input_tokens": 830291.3, "avg_output_tokens": 39597.2, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 9.333, "latency": 3013.297, "stderr": 4.736, "cost_per_test": 1.04722, "avg_input_tokens": 3094893.0, "avg_output_tokens": 102581.7, "avg_cache_read_tokens": 16786.3, "avg_cache_write_tokens": 2486.0, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-4.6": { "accuracy": 7.667, "latency": 8101.384, "stderr": 4.937, "cost_per_test": 1.00872, "avg_input_tokens": 2313626.8, "avg_output_tokens": 56774.2, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 7.667, "latency": 10702.953, "stderr": 3.82, "cost_per_test": 0.091116, "avg_input_tokens": 366791.7, "avg_output_tokens": 35514.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-codex": { "accuracy": 7.333, "latency": 2128.396, "stderr": 6.694, "cost_per_test": 0.151756, "avg_input_tokens": 43436.5, "avg_output_tokens": 9746.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 7.0, "latency": 507.95, "stderr": 4.989, "cost_per_test": 0.044368, "avg_input_tokens": 80990.7, "avg_output_tokens": 12060.3, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "high", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 6.364, "latency": 6323.238, "stderr": 1.819, "cost_per_test": 19.617569, "avg_input_tokens": 1138070.6, "avg_output_tokens": 33953.5, "avg_reasoning_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 6.167, "latency": 1916.743, "stderr": 5.629, "cost_per_test": 0.316446, "avg_input_tokens": 892300.8, "avg_output_tokens": 127767.2, "avg_reasoning_tokens": 117769.3, "avg_cache_read_tokens": 720725.3, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 5.833, "latency": 10438.736, "stderr": 2.466, "cost_per_test": 2.161642, "avg_input_tokens": 1809559.0, "avg_output_tokens": 70416.7, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 5.833, "latency": 28418.598, "stderr": 2.833, "cost_per_test": 1.494618, "avg_input_tokens": 4339753.0, "avg_output_tokens": 82476.8, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 5.667, "latency": 1702.552, "stderr": 3.469, "cost_per_test": 0.042869, "avg_input_tokens": 178307.0, "avg_output_tokens": 14415.3, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-code-fast-1": { "accuracy": 5.333, "latency": 1300.636, "stderr": 1.284, "cost_per_test": 1.203294, "avg_input_tokens": 637537.2, "avg_output_tokens": 717191.3, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 4.667, "latency": 5483.31, "stderr": 2.6, "cost_per_test": 0.405159, "avg_input_tokens": 2398055.3, "avg_output_tokens": 79154.7, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-4.7": { "accuracy": 4.667, "latency": 7018.207, "stderr": 3.429, "cost_per_test": 0.502413, "avg_input_tokens": 1847492.8, "avg_output_tokens": 48790.8, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 4.0, "latency": 3001.342, "stderr": 1.269, "cost_per_test": 1.636035, "avg_input_tokens": 462511.7, "avg_output_tokens": 16566.7, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 3.833, "latency": 5381.491, "stderr": 2.692, "cost_per_test": 0.521296, "avg_input_tokens": 529220.8, "avg_output_tokens": 24479.8, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 3.333, "latency": 4488.047, "stderr": 1.408, "cost_per_test": 2.697792, "avg_input_tokens": 791843.3, "avg_output_tokens": 21484.2, "avg_reasoning_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.5": { "accuracy": 2.667, "latency": 2499.873, "stderr": 1.24, "cost_per_test": 0.790922, "avg_input_tokens": 1051774.3, "avg_output_tokens": 72662.3, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 2.0, "latency": 584.039, "stderr": 0.77, "cost_per_test": 0.197548, "avg_input_tokens": 57871.6, "avg_output_tokens": 30429.4, "avg_cache_write_tokens": 0.0, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 2.0, "latency": 1559.318, "stderr": 1.179, "cost_per_test": 2.309216, "avg_input_tokens": 2204779.5, "avg_output_tokens": 34812.3, "avg_reasoning_tokens": 0.0, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 1.667, "latency": 198.312, "stderr": 0.962, "cost_per_test": 1.106313, "avg_input_tokens": 1217943.8, "avg_output_tokens": 11292.5, "avg_reasoning_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 1.5, "latency": 848.205, "stderr": 0.874, "cost_per_test": 0.080245, "avg_input_tokens": 109533.0, "avg_output_tokens": 116677.3, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 1.5, "latency": 5786.766, "stderr": 0.874, "cost_per_test": 2.20173, "avg_input_tokens": 4085677.3, "avg_output_tokens": 105927.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/labs-devstral-small-2512": { "accuracy": 1.333, "latency": 14075.052, "stderr": 1.217, "cost_per_test": 0.497513, "avg_input_tokens": 4720891.5, "avg_output_tokens": 84744.7, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/devstral-2512": { "accuracy": 1.333, "latency": 15405.47, "stderr": 1.217, "cost_per_test": 1.689963, "avg_input_tokens": 3960435.5, "avg_output_tokens": 52894.5, "avg_reasoning_tokens": 0.0, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 0.833, "latency": 1688.096, "stderr": 0.761, "cost_per_test": 4.676308, "avg_input_tokens": 2180178.3, "avg_output_tokens": 63190.3, "avg_cache_read_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash": { "accuracy": 0.0, "latency": 76.555, "stderr": 0.0, "cost_per_test": 0.003315, "avg_input_tokens": 11051.0, "avg_output_tokens": 0.0, "avg_reasoning_tokens": 0.0, "avg_cache_write_tokens": 0.0, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 0.0, "latency": 402.103, "stderr": 0.0, "cost_per_test": 0.152785, "avg_input_tokens": 557443.2, "avg_output_tokens": 34258.5, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } } } }