{ "metadata": { "benchmark": "IOI v1", "slug": "ioi-v1", "description": "Based on the International Olympiad in Informatics", "benchmark_id": "ioi_v1", "family": "ioi", "version": "1", "updated": "2026-08-09", "dataset_type": "public", "industry": "coding", "tasks": { "overall": "Overall", "ioi2024": "IOI 2024", "ioi2025": "IOI 2025" }, "models": [ "alibaba/qwen3-max", "alibaba/qwen3-max-2026-01-23", "alibaba/qwen3-max-preview", "alibaba/qwen3.7-max", "alibaba/qwen3.8-max", "anthropic/claude-fable-5", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-5", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929-thinking", "deepseek/deepseek-v4-pro", "fireworks/deepseek-v3-0324", "fireworks/deepseek-v3p2", "fireworks/deepseek-v3p2-thinking", "fireworks/qwen3-235b-a22b", "google/gemini-2.5-flash", "google/gemini-2.5-pro", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.5-flash-lite", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-code-fast-1", "kimi/kimi-k2.5-thinking", "meta/muse_spark_1_2", "minimax/MiniMax-M2.1", "minimax/MiniMax-M2.5", "minimax/MiniMax-M2.7", "mistralai/devstral-2512", "mistralai/labs-devstral-small-2512", "mistralai/magistral-medium-2509", "mistralai/mistral-large-2512", "openai/gpt-5-2025-08-07", "openai/gpt-5-codex", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.1-codex", "openai/gpt-5.1-codex-max", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.3-codex", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/o4-mini-2025-04-16", "together/moonshotai/Kimi-K2-Instruct", "zai/glm-4.5", "zai/glm-4.6", "zai/glm-4.7", "zai/glm-5-thinking" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "custom", "mode": "agentic", "archived": true, "total_models": 62 }, "tasks": { "overall": { "anthropic/claude-opus-5": { "accuracy": 91.667, "latency": 1693.04, "stderr": 8.333, "cost_per_test": 5.541014, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 86.667, "latency": 1310.679, "stderr": 1.667, "cost_per_test": 2.98545, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 73.0, "latency": 3946.139, "stderr": 1.0, "cost_per_test": 2.392116, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 72.917, "latency": 1670.558, "stderr": 5.083, "cost_per_test": 0.262867, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 72.25, "latency": 3318.969, "stderr": 9.357, "cost_per_test": 15.101678, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 67.834, "latency": 2660.886, "stderr": 9.867, "cost_per_test": 3.471332, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 65.25, "latency": 1330.749, "stderr": 0.917, "cost_per_test": 0.97448, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 54.833, "latency": 6739.612, "stderr": 8.318, "cost_per_test": 3.871992, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 49.5, "latency": 4771.52, "stderr": 3.667, "cost_per_test": 6.100817, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 47.084, "latency": 3081.905, "stderr": 8.704, "cost_per_test": 10.486954, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 46.75, "latency": 4380.294, "stderr": 10.263, "cost_per_test": 4.859648, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 43.834, "latency": 3574.257, "stderr": 7.81, "cost_per_test": 2.020025, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 39.084, "latency": 1846.526, "stderr": 10.545, "cost_per_test": 0.375052, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 38.834, "latency": 900.44, "stderr": 11.492, "cost_per_test": 0.331188, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 35.833, "latency": 5373.876, "stderr": 7.139, "cost_per_test": 0.568564, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 30.166, "latency": 896.93, "stderr": 7.489, "cost_per_test": 0.709499, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 26.167, "latency": 882.37, "stderr": 1.167, "cost_per_test": 0.636385, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-0709": { "accuracy": 26.167, "latency": 4265.882, "stderr": 6.763, "cost_per_test": 2.693986, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 23.584, "latency": 7682.383, "stderr": 7.547, "cost_per_test": 3.048735, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "zai/glm-5-thinking": { "accuracy": 22.0, "latency": 6806.938, "stderr": 7.832, "cost_per_test": 3.273674, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 21.5, "latency": 1750.069, "stderr": 7.336, "cost_per_test": 0.368193, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-codex-max": { "accuracy": 21.416, "latency": 956.692, "stderr": 6.995, "cost_per_test": 0.422645, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 20.25, "latency": 8211.492, "stderr": 5.157, "cost_per_test": 3.9235, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 20.0, "latency": 2608.252, "stderr": 4.581, "cost_per_test": 1.629264, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 18.334, "latency": 3315.166, "stderr": 5.918, "cost_per_test": 3.07299, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 17.667, "latency": 3828.386, "stderr": 6.084, "cost_per_test": 0.677261, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 17.084, "latency": 2964.799, "stderr": 6.17, "cost_per_test": 0.615461, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max": { "accuracy": 15.666, "latency": 2045.247, "stderr": 6.352, "cost_per_test": 0.824523, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.3": { "accuracy": 15.334, "latency": 628.816, "stderr": 6.313, "cost_per_test": 0.351754, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 15.25, "latency": 1170.492, "stderr": 6.438, "cost_per_test": 0.042833, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 14.416, "latency": 2384.516, "stderr": 4.496, "cost_per_test": 0.05196, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 13.75, "latency": 2953.233, "stderr": 3.547, "cost_per_test": 1.901446, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 12.516, "latency": 6124.896, "stderr": 4.913, "cost_per_test": 17.585952, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 11.5, "latency": 1408.454, "stderr": 4.388, "cost_per_test": 0.97412, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 10.667, "latency": 1558.619, "stderr": 3.089, "cost_per_test": 0.058713, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5-codex": { "accuracy": 9.75, "latency": 1542.894, "stderr": 7.717, "cost_per_test": 2.669955, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 7.75, "latency": 3372.048, "stderr": 3.165, "cost_per_test": 1.399136, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 7.667, "latency": 7675.292, "stderr": 3.198, "cost_per_test": 0.06955, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-4.7": { "accuracy": 7.584, "latency": 5316.876, "stderr": 3.404, "cost_per_test": 0.443515, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 6.75, "latency": 390.378, "stderr": 3.316, "cost_per_test": 0.039168, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "high", "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 6.666, "latency": 22887.998, "stderr": 2.641, "cost_per_test": 1.501416, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 6.5, "latency": 3253.12, "stderr": 2.096, "cost_per_test": 2.388928, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 6.417, "latency": 1896.804, "stderr": 3.607, "cost_per_test": 0.35057, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 6.166, "latency": 7415.352, "stderr": 2.515, "cost_per_test": 2.133406, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 4.916, "latency": 3242.758, "stderr": 2.379, "cost_per_test": 1.154876, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 4.834, "latency": 480.192, "stderr": 2.355, "cost_per_test": 0.156077, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 4.584, "latency": 2216.287, "stderr": 1.139, "cost_per_test": 1.329832, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.6": { "accuracy": 4.334, "latency": 7105.854, "stderr": 2.485, "cost_per_test": 1.516537, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-code-fast-1": { "accuracy": 4.333, "latency": 1083.862, "stderr": 1.157, "cost_per_test": 1.61801, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 4.0, "latency": 6788.576, "stderr": 1.46, "cost_per_test": 3.073948, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 3.834, "latency": 577.632, "stderr": 2.848, "cost_per_test": 0.12675, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-codex": { "accuracy": 3.666, "latency": 2227.442, "stderr": 3.347, "cost_per_test": 0.176857, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 3.084, "latency": 1533.288, "stderr": 1.749, "cost_per_test": 0.02997, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-4.5": { "accuracy": 2.917, "latency": 2336.43, "stderr": 0.987, "cost_per_test": 0.942275, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-2.5-flash": { "accuracy": 2.611, "latency": 350.186, "stderr": 1.03, "cost_per_test": 0.036998, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/labs-devstral-small-2512": { "accuracy": 2.5, "latency": 7466.603, "stderr": 1.781, "cost_per_test": 0.50569, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 2.334, "latency": 5210.402, "stderr": 1.3, "cost_per_test": 0.319345, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 1.667, "latency": 171.479, "stderr": 0.9, "cost_per_test": 0.606484, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 1.25, "latency": 1551.101, "stderr": 0.632, "cost_per_test": 2.367573, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 0.666, "latency": 2211.783, "stderr": 0.444, "cost_per_test": 6.344222, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/devstral-2512": { "accuracy": 0.666, "latency": 8044.645, "stderr": 0.608, "cost_per_test": 1.753753, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 0.0, "latency": 352.565, "stderr": 0.0, "cost_per_test": 0.109574, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "ioi2024": { "anthropic/claude-opus-5": { "accuracy": 100.0, "latency": 1737.255, "stderr": 0.0, "cost_per_test": 6.45987, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 100.0, "latency": 2540.44, "stderr": 0.0, "cost_per_test": 15.68634, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 85.0, "latency": 1314.969, "stderr": 13.693, "cost_per_test": 3.002041, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 78.0, "latency": 1730.527, "stderr": 13.904, "cost_per_test": 0.260799, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 72.0, "latency": 3819.907, "stderr": 13.392, "cost_per_test": 1.999003, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 67.167, "latency": 2241.026, "stderr": 14.878, "cost_per_test": 3.371664, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 66.167, "latency": 678.342, "stderr": 14.997, "cost_per_test": 0.818561, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 58.167, "latency": 2631.97, "stderr": 14.481, "cost_per_test": 1.818037, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 53.167, "latency": 2569.514, "stderr": 14.854, "cost_per_test": 9.010059, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 45.833, "latency": 5252.146, "stderr": 15.842, "cost_per_test": 7.024904, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 43.833, "latency": 7378.658, "stderr": 12.379, "cost_per_test": 4.02342, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 41.0, "latency": 2307.779, "stderr": 17.098, "cost_per_test": 0.401804, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 40.5, "latency": 4033.028, "stderr": 17.613, "cost_per_test": 4.69984, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 38.5, "latency": 1079.286, "stderr": 17.904, "cost_per_test": 0.328592, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 31.333, "latency": 5262.744, "stderr": 10.426, "cost_per_test": 0.580726, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 25.0, "latency": 1087.865, "stderr": 14.652, "cost_per_test": 0.708425, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5-thinking": { "accuracy": 23.667, "latency": 7231.432, "stderr": 11.716, "cost_per_test": 2.864289, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-0709": { "accuracy": 23.167, "latency": 4824.866, "stderr": 11.985, "cost_per_test": 1.953434, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 22.5, "latency": 622.262, "stderr": 12.057, "cost_per_test": 0.654763, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 21.5, "latency": 1069.464, "stderr": 12.27, "cost_per_test": 0.415113, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 20.667, "latency": 11576.195, "stderr": 11.43, "cost_per_test": 2.959818, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 19.0, "latency": 2159.535, "stderr": 10.263, "cost_per_test": 0.842034, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 19.0, "latency": 2583.917, "stderr": 10.263, "cost_per_test": 2.711026, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 18.667, "latency": 5926.555, "stderr": 9.657, "cost_per_test": 15.554335, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 18.0, "latency": 4779.544, "stderr": 5.452, "cost_per_test": 3.923563, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-5.1-codex-max": { "accuracy": 14.5, "latency": 975.205, "stderr": 9.757, "cost_per_test": 0.49262, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 14.167, "latency": 967.635, "stderr": 6.269, "cost_per_test": 0.035235, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 14.0, "latency": 1615.658, "stderr": 6.364, "cost_per_test": 0.052688, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.3": { "accuracy": 13.0, "latency": 555.026, "stderr": 6.272, "cost_per_test": 0.32626, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 12.333, "latency": 3278.996, "stderr": 5.124, "cost_per_test": 1.949747, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 12.167, "latency": 3195.697, "stderr": 5.041, "cost_per_test": 0.561257, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 11.833, "latency": 1702.974, "stderr": 6.126, "cost_per_test": 0.363369, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 11.667, "latency": 1362.606, "stderr": 5.728, "cost_per_test": 2.276975, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 11.167, "latency": 1374.876, "stderr": 5.32, "cost_per_test": 0.050024, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 11.0, "latency": 2627.572, "stderr": 4.967, "cost_per_test": 1.870345, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.7": { "accuracy": 10.5, "latency": 3615.545, "stderr": 5.882, "cost_per_test": 0.384617, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 9.667, "latency": 2018.192, "stderr": 3.949, "cost_per_test": 2.080064, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 9.333, "latency": 1370.235, "stderr": 4.305, "cost_per_test": 1.353118, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 7.667, "latency": 376.345, "stderr": 4.647, "cost_per_test": 0.114606, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 7.667, "latency": 4647.63, "stderr": 5.13, "cost_per_test": 0.047983, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 7.5, "latency": 17357.397, "stderr": 4.458, "cost_per_test": 1.508215, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 6.667, "latency": 1876.864, "stderr": 4.513, "cost_per_test": 0.384694, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 6.5, "latency": 272.805, "stderr": 4.37, "cost_per_test": 0.033967, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "high", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 6.5, "latency": 4391.967, "stderr": 4.383, "cost_per_test": 2.10517, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 6.5, "latency": 7790.385, "stderr": 2.786, "cost_per_test": 3.946165, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 6.167, "latency": 307.059, "stderr": 5.629, "cost_per_test": 0.173256, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash": { "accuracy": 5.222, "latency": 623.818, "stderr": 2.059, "cost_per_test": 0.07068, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 5.167, "latency": 1431.232, "stderr": 1.892, "cost_per_test": 1.023628, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/labs-devstral-small-2512": { "accuracy": 3.667, "latency": 858.154, "stderr": 3.347, "cost_per_test": 0.513867, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-code-fast-1": { "accuracy": 3.333, "latency": 867.087, "stderr": 1.925, "cost_per_test": 2.032726, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-4.5": { "accuracy": 3.167, "latency": 2172.987, "stderr": 1.535, "cost_per_test": 1.093628, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5-codex": { "accuracy": 2.833, "latency": 1650.37, "stderr": 2.586, "cost_per_test": 2.583712, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 1.667, "latency": 144.646, "stderr": 1.521, "cost_per_test": 0.106655, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-4.6": { "accuracy": 1.0, "latency": 6110.323, "stderr": 0.577, "cost_per_test": 2.024354, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 0.5, "latency": 1364.023, "stderr": 0.456, "cost_per_test": 0.01707, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 0.5, "latency": 1542.884, "stderr": 0.456, "cost_per_test": 2.42593, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 0.5, "latency": 2735.47, "stderr": 0.456, "cost_per_test": 8.012136, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.5, "latency": 3472.218, "stderr": 0.456, "cost_per_test": 1.262533, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 0.0, "latency": 303.027, "stderr": 0.0, "cost_per_test": 0.066363, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/devstral-2512": { "accuracy": 0.0, "latency": 683.82, "stderr": 0.0, "cost_per_test": 1.817543, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.1-codex": { "accuracy": 0.0, "latency": 2326.487, "stderr": 0.0, "cost_per_test": 0.201958, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 0.0, "latency": 4937.493, "stderr": 0.0, "cost_per_test": 0.233531, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null } }, "ioi2025": { "openai/gpt-5.6-sol": { "accuracy": 88.333, "latency": 1306.389, "stderr": 10.65, "cost_per_test": 2.96886, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 83.333, "latency": 1648.826, "stderr": 16.667, "cost_per_test": 4.438387, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 74.0, "latency": 4072.372, "stderr": 15.882, "cost_per_test": 2.785229, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 68.5, "latency": 3080.746, "stderr": 12.965, "cost_per_test": 3.571001, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 67.833, "latency": 1610.589, "stderr": 15.511, "cost_per_test": 0.264936, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 65.833, "latency": 6100.567, "stderr": 11.114, "cost_per_test": 3.720564, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 64.333, "latency": 1983.157, "stderr": 12.199, "cost_per_test": 1.130399, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 53.167, "latency": 4290.894, "stderr": 14.321, "cost_per_test": 5.176731, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 53.0, "latency": 4727.561, "stderr": 10.541, "cost_per_test": 5.019455, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 44.5, "latency": 4097.498, "stderr": 18.714, "cost_per_test": 14.517016, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 41.0, "latency": 3594.296, "stderr": 9.077, "cost_per_test": 11.963848, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 40.333, "latency": 5485.007, "stderr": 9.754, "cost_per_test": 0.556402, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 39.167, "latency": 721.594, "stderr": 14.411, "cost_per_test": 0.333784, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 37.833, "latency": 1171.599, "stderr": 8.886, "cost_per_test": 0.764235, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 37.167, "latency": 1385.274, "stderr": 12.346, "cost_per_test": 0.3483, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 31.167, "latency": 1797.164, "stderr": 13.332, "cost_per_test": 0.373017, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 29.5, "latency": 4516.544, "stderr": 5.854, "cost_per_test": 2.222013, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-0709": { "accuracy": 29.167, "latency": 3706.897, "stderr": 6.269, "cost_per_test": 3.434538, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 29.0, "latency": 2588.931, "stderr": 7.699, "cost_per_test": 1.388182, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-codex-max": { "accuracy": 28.333, "latency": 938.178, "stderr": 10.027, "cost_per_test": 0.35267, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 27.333, "latency": 676.876, "stderr": 10.257, "cost_per_test": 0.564344, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 26.5, "latency": 3788.571, "stderr": 9.857, "cost_per_test": 3.137652, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 23.167, "latency": 4461.074, "stderr": 11.074, "cost_per_test": 0.793265, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 22.5, "latency": 11643.441, "stderr": 8.755, "cost_per_test": 3.923438, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "zai/glm-5-thinking": { "accuracy": 20.333, "latency": 6382.444, "stderr": 10.397, "cost_per_test": 3.68306, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.3": { "accuracy": 17.667, "latency": 702.606, "stderr": 10.959, "cost_per_test": 0.377249, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 17.667, "latency": 4046.414, "stderr": 5.896, "cost_per_test": 3.434954, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-codex": { "accuracy": 16.667, "latency": 1435.417, "stderr": 15.215, "cost_per_test": 2.756198, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 16.333, "latency": 1373.348, "stderr": 11.246, "cost_per_test": 0.050431, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3-max-2026-01-23": { "accuracy": 15.167, "latency": 2627.47, "stderr": 4.907, "cost_per_test": 1.853145, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-pro": { "accuracy": 15.167, "latency": 3770.063, "stderr": 6.853, "cost_per_test": 0.388888, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 14.833, "latency": 3153.373, "stderr": 6.353, "cost_per_test": 0.051233, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "none", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 13.667, "latency": 1446.674, "stderr": 7.647, "cost_per_test": 0.595122, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 10.167, "latency": 1742.362, "stderr": 3.14, "cost_per_test": 0.067402, "temperature": 1.0, "top_p": null, "max_output_tokens": 20480, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 9.833, "latency": 3021.03, "stderr": 3.295, "cost_per_test": 1.233933, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 9.333, "latency": 3013.297, "stderr": 4.736, "cost_per_test": 1.04722, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-4.6": { "accuracy": 7.667, "latency": 8101.384, "stderr": 4.937, "cost_per_test": 1.00872, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 7.667, "latency": 10702.953, "stderr": 3.82, "cost_per_test": 0.091116, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-codex": { "accuracy": 7.333, "latency": 2128.396, "stderr": 6.694, "cost_per_test": 0.151756, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 7.0, "latency": 507.95, "stderr": 4.989, "cost_per_test": 0.044368, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "high", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 6.364, "latency": 6323.238, "stderr": 1.819, "cost_per_test": 19.617569, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 6.167, "latency": 1916.743, "stderr": 5.629, "cost_per_test": 0.316446, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 5.833, "latency": 10438.736, "stderr": 2.466, "cost_per_test": 2.161642, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 5.833, "latency": 28418.598, "stderr": 2.833, "cost_per_test": 1.494618, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 5.667, "latency": 1702.552, "stderr": 3.469, "cost_per_test": 0.042869, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-code-fast-1": { "accuracy": 5.333, "latency": 1300.636, "stderr": 1.284, "cost_per_test": 1.203294, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 4.667, "latency": 5483.31, "stderr": 2.6, "cost_per_test": 0.405159, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-4.7": { "accuracy": 4.667, "latency": 7018.207, "stderr": 3.429, "cost_per_test": 0.502413, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 4.0, "latency": 3001.342, "stderr": 1.269, "cost_per_test": 1.636035, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 3.833, "latency": 5381.491, "stderr": 2.692, "cost_per_test": 0.521296, "temperature": 0.7, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 3.333, "latency": 4488.047, "stderr": 1.408, "cost_per_test": 2.697792, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.5": { "accuracy": 2.667, "latency": 2499.873, "stderr": 1.24, "cost_per_test": 0.790922, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 2.0, "latency": 584.039, "stderr": 0.77, "cost_per_test": 0.197548, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 2.0, "latency": 1559.318, "stderr": 1.179, "cost_per_test": 2.309216, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 1.667, "latency": 198.312, "stderr": 0.962, "cost_per_test": 1.106313, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 1.5, "latency": 848.205, "stderr": 0.874, "cost_per_test": 0.080245, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 1.5, "latency": 5786.766, "stderr": 0.874, "cost_per_test": 2.20173, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/labs-devstral-small-2512": { "accuracy": 1.333, "latency": 14075.052, "stderr": 1.217, "cost_per_test": 0.497513, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/devstral-2512": { "accuracy": 1.333, "latency": 15405.47, "stderr": 1.217, "cost_per_test": 1.689963, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 0.833, "latency": 1688.096, "stderr": 0.761, "cost_per_test": 4.676308, "temperature": 0.7, "top_p": 1.0, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash": { "accuracy": 0.0, "latency": 76.555, "stderr": 0.0, "cost_per_test": 0.003315, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 0.0, "latency": 402.103, "stderr": 0.0, "cost_per_test": 0.152785, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } } } }