{ "metadata": { "benchmark": "LiveCodeBench", "slug": "lcb", "description": "Our Implementation of the LiveCodeBench benchmark", "benchmark_id": "lcb", "family": "lcb", "version": "1", "updated": "2026-09-01", "dataset_type": "public", "industry": "coding", "tasks": { "overall": "Overall", "easy": "Easy", "medium": "Medium", "hard": "Hard" }, "models": [ "ai21labs/jamba-large-1.6", "ai21labs/jamba-mini-1.6", "alibaba/qwen3-max", "alibaba/qwen3-max-preview", "alibaba/qwen3.5-flash", "alibaba/qwen3.5-plus-thinking", "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "ant/ling-3.0-flash-2607", "anthropic/claude-3-5-haiku-20241022", "anthropic/claude-3-5-sonnet-20241022", "anthropic/claude-3-7-sonnet-20250219", "anthropic/claude-3-7-sonnet-20250219-thinking", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-opus-4-1-20250805", "anthropic/claude-opus-4-1-20250805-thinking", "anthropic/claude-opus-4-20250514", "anthropic/claude-opus-4-20250514-thinking", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-thinking", "anthropic/claude-opus-4-6-thinking", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-20250514-thinking", "anthropic/claude-sonnet-4-5-20250929-thinking", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "cohere/command-a-03-2025", "cohere/command-r-plus", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "fireworks/deepseek-r1", "fireworks/deepseek-v3-0324", "fireworks/deepseek-v3p2", "fireworks/deepseek-v3p2-thinking", "fireworks/gpt-oss-120b", "fireworks/gpt-oss-20b", "fireworks/llama4-maverick-instruct-basic", "fireworks/qwen3-235b-a22b", "google/gemini-1.5-flash-002", "google/gemini-1.5-pro-002", "google/gemini-2.0-flash-001", "google/gemini-2.5-flash-lite-preview-09-2025", "google/gemini-2.5-flash-lite-preview-09-2025-thinking", "google/gemini-2.5-flash-preview-04-17", "google/gemini-2.5-flash-preview-04-17-thinking", "google/gemini-2.5-flash-preview-09-2025", "google/gemini-2.5-flash-preview-09-2025-thinking", "google/gemini-2.5-pro-preview-03-25", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "grok/grok-2-1212", "grok/grok-3", "grok/grok-3-mini-fast-high-reasoning", "grok/grok-3-mini-fast-low-reasoning", "grok/grok-4-0709", "grok/grok-4-1-fast-non-reasoning", "grok/grok-4-1-fast-reasoning", "grok/grok-4-fast-non-reasoning", "grok/grok-4-fast-reasoning", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-code-fast-1", "kimi/kimi-k2-thinking", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k2.7-code", "kimi/kimi-k3", "meta/muse_spark_1_1", "minimax/MiniMax-M2.1", "minimax/MiniMax-M2.5", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/devstral-2512", "mistralai/labs-devstral-small-2512", "mistralai/magistral-medium-2509", "mistralai/magistral-small-2509", "mistralai/mistral-large-2411", "mistralai/mistral-large-2512", "mistralai/mistral-medium-2505", "mistralai/mistral-small-2402", "mistralai/mistral-small-2503", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-4.1-2025-04-14", "openai/gpt-4.1-mini-2025-04-14", "openai/gpt-4.1-nano-2025-04-14", "openai/gpt-4o-2024-11-20", "openai/gpt-4o-mini-2024-07-18", "openai/gpt-5-2025-08-07", "openai/gpt-5-codex", "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-nano-2025-08-07", "openai/gpt-5.1-2025-11-13", "openai/gpt-5.1-codex", "openai/gpt-5.1-codex-max", "openai/gpt-5.2-2025-12-11", "openai/gpt-5.2-codex", "openai/gpt-5.3-codex", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/o1-2024-12-17", "openai/o3-2025-04-16", "openai/o3-mini-2025-01-31", "openai/o4-mini-2025-04-16", "poolside/laguna-m.1", "poolside/laguna-xs.2", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561", "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking", "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "together/meta-llama/Llama-4-Scout-17B-16E-Instruct", "together/moonshotai/Kimi-K2-Instruct", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "zai/glm-4.5", "zai/glm-4.6", "zai/glm-4.7", "zai/glm-5-thinking", "zai/glm-5.1", "zai/glm-5.2", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": false, "runner": "platform", "mode": "one-shot", "archived": false, "partner": false, "total_models": 143 }, "tasks": { "overall": { "anthropic/claude-fable-5-1": { "accuracy": 90.515, "latency": 59.738, "stderr": 0.863, "cost_per_test": 0.260604, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 89.778, "latency": 118.53, "stderr": 0.892, "cost_per_test": 0.42816, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 89.484, "latency": 24.07, "stderr": 0.897, "cost_per_test": 0.07124, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 89.033, "latency": 59.22, "stderr": 0.913, "cost_per_test": 0.134205, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 88.652, "latency": 14.533, "stderr": 0.924, "cost_per_test": 0.040218, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 88.485, "latency": 88.891, "stderr": 0.931, "cost_per_test": 0.101147, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 88.224, "latency": 118.4, "stderr": 0.941, "cost_per_test": 0.039923, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 88.08, "latency": 31.883, "stderr": 0.939, "cost_per_test": 0.056383, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-codex": { "accuracy": 87.993, "latency": 168.271, "stderr": 0.942, "cost_per_test": 0.11667, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 87.852, "latency": 337.788, "stderr": 0.951, "cost_per_test": 0.091412, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 87.819, "latency": 159.595, "stderr": 0.948, "cost_per_test": 0.327373, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 87.604, "latency": 64.371, "stderr": 0.95, "cost_per_test": 0.085937, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 87.525, "latency": 273.233, "stderr": 0.958, "cost_per_test": 0.063876, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 87.484, "latency": 715.504, "stderr": 0.953, "cost_per_test": 0.109669, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.5": { "accuracy": 87.351, "latency": 589.356, "stderr": 0.961, "cost_per_test": 0.045235, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 87.313, "latency": 132.631, "stderr": 0.968, "cost_per_test": 0.091585, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 87.264, "latency": 126.61, "stderr": 0.968, "cost_per_test": 0.020251, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 87.185, "latency": 200.263, "stderr": 0.971, "cost_per_test": 0.103719, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 87.057, "latency": 192.175, "stderr": 0.974, "cost_per_test": 0.086751, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 86.771, "latency": 529.063, "stderr": 0.972, "cost_per_test": 0.06543, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 86.605, "latency": 33.673, "stderr": 0.979, "cost_per_test": 0.01153, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 86.486, "latency": 141.24, "stderr": 0.983, "cost_per_test": 0.125706, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 86.407, "latency": 87.226, "stderr": 0.982, "cost_per_test": 0.12729, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 85.976, "latency": 161.2, "stderr": 0.992, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 85.952, "latency": 475.705, "stderr": 0.977, "cost_per_test": 0.077628, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 85.931, "latency": 302.324, "stderr": 0.992, "cost_per_test": 0.020896, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 85.93, "latency": 44.331, "stderr": 1.017, "cost_per_test": 0.031103, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 85.911, "latency": 150.441, "stderr": 0.974, "cost_per_test": 0.063751, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 85.906, "latency": 113.253, "stderr": 0.992, "cost_per_test": 0.046497, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 85.591, "latency": 91.931, "stderr": 1.004, "cost_per_test": 0.050525, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.1-codex": { "accuracy": 85.55, "latency": 233.557, "stderr": 1.003, "cost_per_test": 0.137502, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 85.517, "latency": 257.712, "stderr": 1.003, "cost_per_test": 0.153251, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 85.361, "latency": 151.513, "stderr": 1.027, "cost_per_test": 0.135162, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 85.326, "latency": 247.472, "stderr": 1.001, "cost_per_test": 0.04786, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.5": { "accuracy": 85.296, "latency": 166.845, "stderr": 1.018, "cost_per_test": 0.0666, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 85.073, "latency": 461.91, "stderr": 1.006, "cost_per_test": 0.350587, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5-codex": { "accuracy": 84.725, "latency": 134.963, "stderr": 1.032, "cost_per_test": 0.097451, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 84.676, "latency": 324.486, "stderr": 1.02, "cost_per_test": 0.34344, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 84.494, "latency": 3248.962, "stderr": 1.024, "cost_per_test": 0.020464, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 84.265, "latency": 76.09, "stderr": 1.028, "cost_per_test": 0.05249, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 84.141, "latency": 209.424, "stderr": 1.04, "cost_per_test": 0.329246, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 84.009, "latency": 60.089, "stderr": 1.044, "cost_per_test": 0.002468, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 83.996, "latency": 149.298, "stderr": 1.035, "cost_per_test": 0.044983, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 83.988, "latency": 60.397, "stderr": 1.038, "cost_per_test": 0.003019, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 83.914, "latency": 63.948, "stderr": 1.03, "cost_per_test": 0.032963, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 83.868, "latency": 200.023, "stderr": 1.03, "cost_per_test": 0.038656, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 83.67, "latency": 164.427, "stderr": 1.037, "cost_per_test": 0.971, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "openai/gpt-5.1-codex-max": { "accuracy": 83.558, "latency": 66.201, "stderr": 1.055, "cost_per_test": 0.001556, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 83.28, "latency": 209.294, "stderr": 1.04, "cost_per_test": 0.009542, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4-0709": { "accuracy": 83.247, "latency": 229.402, "stderr": 1.026, "cost_per_test": 0.155988, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 83.234, "latency": 81.703, "stderr": 1.055, "cost_per_test": 0.008486, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 82.604, "latency": 56.513, "stderr": 1.088, "cost_per_test": 0.089831, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 82.429, "latency": 77.021, "stderr": 1.088, "cost_per_test": 0.094655, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-4.7": { "accuracy": 82.234, "latency": 393.276, "stderr": 0.993, "cost_per_test": 0.035469, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 82.208, "latency": 32.84, "stderr": 1.044, "cost_per_test": 0.013647, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 82.154, "latency": 366.506, "stderr": 1.055, "cost_per_test": 0.03743, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 82.091, "latency": 295.009, "stderr": 1.063, "cost_per_test": 0.370575, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 82.05, "latency": 214.178, "stderr": 1.071, "cost_per_test": 0.027349, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 81.868, "latency": 289.752, "stderr": 1.057, "cost_per_test": 0.043918, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 81.756, "latency": 246.072, "stderr": 1.099, "cost_per_test": 0.018485, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 81.512, "latency": 108.607, "stderr": 1.072, "cost_per_test": 0.002243, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 81.465, "latency": 188.757, "stderr": 1.094, "cost_per_test": 0.032935, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 81.38, "latency": 226.424, "stderr": 1.061, "cost_per_test": 0.045983, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 81.354, "latency": 271.074, "stderr": 1.069, "cost_per_test": 0.006808, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-4.6": { "accuracy": 81.036, "latency": 236.176, "stderr": 1.082, "cost_per_test": 0.026909, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 80.695, "latency": 254.276, "stderr": 1.099, "cost_per_test": 0.003266, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 80.641, "latency": 103.449, "stderr": 1.091, "cost_per_test": 0.003892, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 80.535, "latency": 246.75, "stderr": 1.065, "cost_per_test": 0.069397, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 80.51, "latency": 301.901, "stderr": 1.051, "cost_per_test": 0.007053, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 80.387, "latency": 109.453, "stderr": 1.078, "cost_per_test": 0.006706, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 80.116, "latency": 30.749, "stderr": 1.083, "cost_per_test": 0.00435, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 79.926, "latency": 318.934, "stderr": 1.096, "cost_per_test": 0.02142, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 79.208, "latency": 312.011, "stderr": 1.119, "cost_per_test": 0.033052, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-2.5-pro-preview-03-25": { "accuracy": 79.164, "latency": 164.657, "stderr": 1.086, "cost_per_test": 0.02004, "temperature": 1.0, "top_p": null, "max_output_tokens": 32512, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 79.014, "latency": 21.084, "stderr": 1.098, "cost_per_test": 0.019272, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 78.973, "latency": 51.14, "stderr": 1.1, "cost_per_test": 0.004507, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 78.215, "latency": 107.845, "stderr": 1.108, "cost_per_test": 0.021213, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 76.22, "latency": 213.662, "stderr": 1.084, "cost_per_test": 0.008057, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 76.214, "latency": 48.671, "stderr": 1.136, "cost_per_test": 0.002383, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 75.063, "latency": 46.785, "stderr": 1.13, "cost_per_test": 0.002367, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 75.034, "latency": 13.599, "stderr": 1.12, "cost_per_test": 0.077323, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 74.86, "latency": 209.97, "stderr": 1.132, "cost_per_test": 0.049906, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 72.996, "latency": 109.547, "stderr": 1.125, "cost_per_test": 0.135104, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 72.131, "latency": 100.782, "stderr": 1.123, "cost_per_test": 0.015218, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 71.484, "latency": 53.8, "stderr": 1.154, "cost_per_test": 0.033121, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 71.385, "latency": 38.697, "stderr": 1.138, "cost_per_test": 0.000408, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 70.62, "latency": 429.483, "stderr": 1.145, "cost_per_test": 0.011344, "temperature": 1.0, "top_p": null, "max_output_tokens": 38912, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 70.449, "latency": 66.651, "stderr": 1.159, "cost_per_test": 0.006889, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 70.221, "latency": 86.074, "stderr": 1.15, "cost_per_test": 0.040788, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 70.216, "latency": 136.359, "stderr": 0.927, "cost_per_test": 0.005053, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-20250514-thinking": { "accuracy": 70.188, "latency": 93.538, "stderr": 1.111, "cost_per_test": 0.397382, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 69.856, "latency": 39.343, "stderr": 1.198, "cost_per_test": 0.000677, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.2": { "accuracy": 69.5, "latency": 532.222, "stderr": 1.179, "cost_per_test": 0.129308, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 68.117, "latency": 235.323, "stderr": 1.136, "cost_per_test": 0.005654, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 67.802, "latency": 186.427, "stderr": 1.137, "cost_per_test": 0.002349, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 67.669, "latency": 17.442, "stderr": 1.172, "cost_per_test": 0.002003, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-4.5": { "accuracy": 67.446, "latency": 391.268, "stderr": 1.131, "cost_per_test": 0.018278, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 66.91, "latency": 96.247, "stderr": 1.167, "cost_per_test": 0.01353, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 66.456, "latency": 92.167, "stderr": 1.125, "cost_per_test": 0.360655, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 66.265, "latency": 30.707, "stderr": 1.141, "cost_per_test": 0.002154, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 65.478, "latency": 26.93, "stderr": 1.164, "cost_per_test": 0.001776, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 64.559, "latency": 32.509, "stderr": 1.112, "cost_per_test": 0.055988, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 63.145, "latency": 1150.624, "stderr": 1.157, "cost_per_test": 0.033631, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 62.629, "latency": 14.777, "stderr": 1.117, "cost_per_test": 0.055697, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 62.392, "latency": 42.611, "stderr": 1.125, "cost_per_test": 0.042318, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-code-fast-1": { "accuracy": 61.969, "latency": 25.838, "stderr": 1.203, "cost_per_test": 0.008528, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 60.436, "latency": 96.471, "stderr": 1.138, "cost_per_test": 0.124735, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 59.673, "latency": 14.058, "stderr": 1.168, "cost_per_test": 0.013812, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 58.369, "latency": 152.4, "stderr": 1.172, "cost_per_test": null, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 58.158, "latency": 27.592, "stderr": 1.181, "cost_per_test": 0.001178, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 56.936, "latency": 16.407, "stderr": 1.176, "cost_per_test": 0.011721, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 56.662, "latency": 16.232, "stderr": 1.146, "cost_per_test": 0.020551, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 55.337, "latency": 28.888, "stderr": 1.152, "cost_per_test": 0.002215, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 54.666, "latency": 30.565, "stderr": 1.118, "cost_per_test": 0.00931, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3": { "accuracy": 52.901, "latency": 5.517, "stderr": 1.134, "cost_per_test": 0.009356, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/devstral-2512": { "accuracy": 51.841, "latency": 87.57, "stderr": 1.174, "cost_per_test": 0.056631, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 50.264, "latency": 92.082, "stderr": 1.393, "cost_per_test": 0.328067, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 49.628, "latency": 11.035, "stderr": 1.14, "cost_per_test": 0.011661, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 47.251, "latency": 9.442, "stderr": 1.169, "cost_per_test": 0.000832, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-preview-04-17-thinking": { "accuracy": 46.871, "latency": 124.947, "stderr": 1.441, "cost_per_test": 0.001987, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 46.095, "latency": 8.958, "stderr": 1.146, "cost_per_test": 0.001012, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 44.845, "latency": 12.483, "stderr": 1.145, "cost_per_test": 0.00161, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 43.608, "latency": 3.358, "stderr": 1.078, "cost_per_test": 0.000304, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 43.444, "latency": 4.347, "stderr": 1.123, "cost_per_test": 0.003999, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/labs-devstral-small-2512": { "accuracy": 43.178, "latency": 18.821, "stderr": 1.155, "cost_per_test": 0.000346, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 42.718, "latency": 11.801, "stderr": 1.152, "cost_per_test": 0.000298, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 42.622, "latency": 11.962, "stderr": 1.092, "cost_per_test": 0.000436, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 41.918, "latency": 10.581, "stderr": 1.129, "cost_per_test": 0.003064, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 41.719, "latency": 3.954, "stderr": 1.089, "cost_per_test": 0.001975, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 41.175, "latency": 81.561, "stderr": 1.109, "cost_per_test": 0.059767, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-2-1212": { "accuracy": 38.679, "latency": 2.378, "stderr": 1.048, "cost_per_test": 0.003004, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 38.541, "latency": 16.762, "stderr": 1.016, "cost_per_test": 0.000615, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 37.088, "latency": 5.349, "stderr": 1.091, "cost_per_test": 0.003291, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 36.91, "latency": 2.512, "stderr": 1.079, "cost_per_test": 0.000147, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 36.341, "latency": 2.945, "stderr": 1.047, "cost_per_test": 0.000743, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 36.308, "latency": 44.228, "stderr": 1.039, "cost_per_test": null, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 35.071, "latency": 11.746, "stderr": 1.086, "cost_per_test": 0.006568, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 31.815, "latency": 4.007, "stderr": 1.109, "cost_per_test": 0.000147, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 26.423, "latency": 7.364, "stderr": 1.129, "cost_per_test": 0.000298, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 22.325, "latency": 4.84, "stderr": 1.051, "cost_per_test": 0.003236, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "cohere/command-r-plus": { "accuracy": 18.238, "latency": 5.463, "stderr": 1.016, "cost_per_test": 0.003267, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 15.781, "latency": 5.04, "stderr": 0.986, "cost_per_test": 0.000382, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 9.918, "latency": 1.139, "stderr": 0.867, "cost_per_test": 0.000199, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "easy": { "google/gemini-3.7-flash": { "accuracy": 99.068, "latency": 3.213, "stderr": 0.535, "cost_per_test": 0.007966, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 99.068, "latency": 3.727, "stderr": 0.535, "cost_per_test": 0.007871, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 99.068, "latency": 3.764, "stderr": 0.535, "cost_per_test": 0.009821, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 99.068, "latency": 4.841, "stderr": 0.535, "cost_per_test": 0.026073, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 99.068, "latency": 4.888, "stderr": 0.535, "cost_per_test": 0.015514, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 99.068, "latency": 6.74, "stderr": 0.535, "cost_per_test": 0.002057, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 99.068, "latency": 6.914, "stderr": 0.535, "cost_per_test": 0.012428, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-codex": { "accuracy": 99.068, "latency": 10.612, "stderr": 0.535, "cost_per_test": 0.007545, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 99.068, "latency": 12.108, "stderr": 0.535, "cost_per_test": 0.02112, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 99.068, "latency": 12.711, "stderr": 0.535, "cost_per_test": 0.016532, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 99.068, "latency": 12.902, "stderr": 0.535, "cost_per_test": 0.001137, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 99.068, "latency": 13.017, "stderr": 0.535, "cost_per_test": 0.003684, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 99.068, "latency": 13.5, "stderr": 0.535, "cost_per_test": 0.005422, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 99.068, "latency": 16.472, "stderr": 0.535, "cost_per_test": 0.000335, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 99.068, "latency": 17.469, "stderr": 0.535, "cost_per_test": 0.013703, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 99.068, "latency": 19.291, "stderr": 0.535, "cost_per_test": 0.021104, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 99.068, "latency": 22.365, "stderr": 0.535, "cost_per_test": 0.006877, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 99.068, "latency": 22.478, "stderr": 0.535, "cost_per_test": 0.008132, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 99.068, "latency": 24.441, "stderr": 0.535, "cost_per_test": 0.005115, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 99.068, "latency": 30.806, "stderr": 0.535, "cost_per_test": 0.002059, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 99.068, "latency": 38.258, "stderr": 0.535, "cost_per_test": 0.008048, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 99.068, "latency": 39.468, "stderr": 0.535, "cost_per_test": 0.049942, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-20250514-thinking": { "accuracy": 99.068, "latency": 39.894, "stderr": 0.535, "cost_per_test": 0.173558, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 99.068, "latency": 41.655, "stderr": 0.535, "cost_per_test": 0.06154, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 99.068, "latency": 43.905, "stderr": 0.535, "cost_per_test": 0.001487, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 99.068, "latency": 46.546, "stderr": 0.535, "cost_per_test": 0.032344, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 99.068, "latency": 47.032, "stderr": 0.535, "cost_per_test": 0.02567, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-0709": { "accuracy": 99.068, "latency": 50.479, "stderr": 0.535, "cost_per_test": 0.032638, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 99.068, "latency": 76.515, "stderr": 0.535, "cost_per_test": 0.008417, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 99.068, "latency": 171.189, "stderr": 0.535, "cost_per_test": 0.032636, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 99.068, "latency": 205.107, "stderr": 0.535, "cost_per_test": 0.03336, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 99.068, "latency": 449.171, "stderr": 0.535, "cost_per_test": 0.005859, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 98.758, "latency": 3.563, "stderr": 0.617, "cost_per_test": 0.000245, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 98.758, "latency": 5.167, "stderr": 0.617, "cost_per_test": 0.004906, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 98.758, "latency": 7.1, "stderr": 0.617, "cost_per_test": 0.003345, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 98.758, "latency": 10.922, "stderr": 0.617, "cost_per_test": 0.001971, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 98.758, "latency": 12.692, "stderr": 0.617, "cost_per_test": 0.033455, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.1-codex": { "accuracy": 98.758, "latency": 13.338, "stderr": 0.617, "cost_per_test": 0.007355, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 98.758, "latency": 14.231, "stderr": 0.617, "cost_per_test": 0.012439, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 98.758, "latency": 14.6, "stderr": 0.617, "cost_per_test": 0.002031, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 98.758, "latency": 16.263, "stderr": 0.617, "cost_per_test": 0.022982, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 98.758, "latency": 17.082, "stderr": 0.617, "cost_per_test": 0.008187, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 98.758, "latency": 17.314, "stderr": 0.617, "cost_per_test": 0.007149, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3": { "accuracy": 98.758, "latency": 17.924, "stderr": 0.617, "cost_per_test": 0.005812, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 98.758, "latency": 18.116, "stderr": 0.617, "cost_per_test": 0.000707, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 98.758, "latency": 18.448, "stderr": 0.617, "cost_per_test": 0.013965, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 98.758, "latency": 18.54, "stderr": 0.617, "cost_per_test": 0.004698, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 98.758, "latency": 18.987, "stderr": 0.617, "cost_per_test": 0.00869, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 98.758, "latency": 26.601, "stderr": 0.617, "cost_per_test": 0.027167, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 98.758, "latency": 27.107, "stderr": 0.617, "cost_per_test": 0.018785, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 98.758, "latency": 29.279, "stderr": 0.617, "cost_per_test": 0.005942, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 98.758, "latency": 30.579, "stderr": 0.617, "cost_per_test": 0.002727, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 98.758, "latency": 38.083, "stderr": 0.617, "cost_per_test": 0.154263, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 98.758, "latency": 40.099, "stderr": 0.617, "cost_per_test": 0.261576, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 98.758, "latency": 44.105, "stderr": 0.617, "cost_per_test": 0.004989, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 98.758, "latency": 54.06, "stderr": 0.617, "cost_per_test": 0.020032, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 98.758, "latency": 56.68, "stderr": 0.617, "cost_per_test": 0.008151, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 98.758, "latency": 67.765, "stderr": 0.617, "cost_per_test": 0.005632, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 98.758, "latency": 73.491, "stderr": 0.617, "cost_per_test": 0.010782, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 98.758, "latency": 86.637, "stderr": 0.617, "cost_per_test": 0.003773, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-pro-preview-03-25": { "accuracy": 98.758, "latency": 96.699, "stderr": 0.617, "cost_per_test": 0.012934, "temperature": 1.0, "top_p": null, "max_output_tokens": 32512, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 98.758, "latency": 145.34, "stderr": 0.617, "cost_per_test": 0.027081, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.1": { "accuracy": 98.754, "latency": 44.075, "stderr": 0.619, "cost_per_test": 0.008931, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 98.447, "latency": 10.019, "stderr": 0.689, "cost_per_test": 0.005918, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 98.447, "latency": 15.289, "stderr": 0.689, "cost_per_test": 0.018572, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 98.447, "latency": 16.039, "stderr": 0.689, "cost_per_test": 0.011252, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 98.447, "latency": 16.815, "stderr": 0.689, "cost_per_test": 0.022051, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 98.447, "latency": 16.96, "stderr": 0.689, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 98.447, "latency": 21.851, "stderr": 0.689, "cost_per_test": 0.008895, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 98.447, "latency": 26.702, "stderr": 0.689, "cost_per_test": 0.003024, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 98.447, "latency": 34.354, "stderr": 0.689, "cost_per_test": 0.000472, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 98.447, "latency": 39.979, "stderr": 0.689, "cost_per_test": 0.001366, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.5": { "accuracy": 98.447, "latency": 48.393, "stderr": 0.689, "cost_per_test": 0.012472, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 98.447, "latency": 56.858, "stderr": 0.689, "cost_per_test": 0.072221, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-4.5": { "accuracy": 98.447, "latency": 134.586, "stderr": 0.689, "cost_per_test": 0.006886, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.3": { "accuracy": 98.447, "latency": 1725.751, "stderr": 0.689, "cost_per_test": 0.003271, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 98.137, "latency": 4.226, "stderr": 0.754, "cost_per_test": 0.007722, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 98.137, "latency": 6.12, "stderr": 0.754, "cost_per_test": 0.000925, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5-codex": { "accuracy": 98.137, "latency": 8.824, "stderr": 0.754, "cost_per_test": 0.006587, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 98.137, "latency": 9.769, "stderr": 0.754, "cost_per_test": 0.00096, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 98.137, "latency": 10.915, "stderr": 0.754, "cost_per_test": 0.000188, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-4.6": { "accuracy": 98.137, "latency": 25.863, "stderr": 0.754, "cost_per_test": 0.003968, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 98.137, "latency": 26.505, "stderr": 0.754, "cost_per_test": 0.013517, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-4.7": { "accuracy": 98.137, "latency": 39.111, "stderr": 0.754, "cost_per_test": 0.004361, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 98.137, "latency": 43.179, "stderr": 0.754, "cost_per_test": 0.003387, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 98.137, "latency": 70.795, "stderr": 0.754, "cost_per_test": 0.000695, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 98.137, "latency": 297.043, "stderr": 0.754, "cost_per_test": 0.008835, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 97.826, "latency": 11.686, "stderr": 0.813, "cost_per_test": 0.001914, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 97.826, "latency": 14.3, "stderr": 0.813, "cost_per_test": 0.001109, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 97.826, "latency": 15.91, "stderr": 0.813, "cost_per_test": 0.000953, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 97.826, "latency": 18.059, "stderr": 0.813, "cost_per_test": 0.001072, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 97.826, "latency": 47.866, "stderr": 0.813, "cost_per_test": 0.000681, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 97.826, "latency": 70.383, "stderr": 0.813, "cost_per_test": 0.002724, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 97.516, "latency": 3.293, "stderr": 0.867, "cost_per_test": 0.000512, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 97.516, "latency": 8.932, "stderr": 0.867, "cost_per_test": 0.000242, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-codex-max": { "accuracy": 97.516, "latency": 9.963, "stderr": 0.867, "cost_per_test": 0.001916, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 97.516, "latency": 22.583, "stderr": 0.613, "cost_per_test": 0.037777, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "poolside/laguna-m.1": { "accuracy": 97.516, "latency": 128.045, "stderr": 0.867, "cost_per_test": 0.001719, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 97.205, "latency": 4.104, "stderr": 0.919, "cost_per_test": 0.00245, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 97.205, "latency": 17.242, "stderr": 0.919, "cost_per_test": 0.002214, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-5.2": { "accuracy": 97.205, "latency": 57.872, "stderr": 0.919, "cost_per_test": 0.014984, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 96.894, "latency": 8.598, "stderr": 0.967, "cost_per_test": 0.008354, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 96.584, "latency": 6.162, "stderr": 1.012, "cost_per_test": 0.007004, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 96.584, "latency": 13.189, "stderr": 1.012, "cost_per_test": 0.000798, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 96.273, "latency": 7.002, "stderr": 1.056, "cost_per_test": 0.0002, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 96.273, "latency": 9.857, "stderr": 1.056, "cost_per_test": 0.035912, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 96.273, "latency": 36.87, "stderr": 1.056, "cost_per_test": 0.002892, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 96.273, "latency": 142.152, "stderr": 1.056, "cost_per_test": 0.006518, "temperature": 1.0, "top_p": null, "max_output_tokens": 38912, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 95.342, "latency": 2.25, "stderr": 1.174, "cost_per_test": 0.000377, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-code-fast-1": { "accuracy": 95.342, "latency": 5.086, "stderr": 1.174, "cost_per_test": 0.00258, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 95.342, "latency": 8.113, "stderr": 1.174, "cost_per_test": 0.00075, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 95.031, "latency": 72.746, "stderr": 1.211, "cost_per_test": null, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 94.72, "latency": 1.696, "stderr": 1.246, "cost_per_test": 0.001012, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 94.72, "latency": 2.191, "stderr": 1.246, "cost_per_test": 0.001899, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-3": { "accuracy": 94.099, "latency": 2.791, "stderr": 1.313, "cost_per_test": 0.005439, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 92.236, "latency": 8.78, "stderr": 1.491, "cost_per_test": 0.008858, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 90.683, "latency": 1.313, "stderr": 1.62, "cost_per_test": 0.000123, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/devstral-2512": { "accuracy": 90.683, "latency": 19.735, "stderr": 1.62, "cost_per_test": 0.004242, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 88.82, "latency": 3.173, "stderr": 1.756, "cost_per_test": 0.000199, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 88.199, "latency": 2.197, "stderr": 1.798, "cost_per_test": 0.001189, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 88.199, "latency": 2.443, "stderr": 1.798, "cost_per_test": 0.00056, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 87.888, "latency": 6.489, "stderr": 1.818, "cost_per_test": 0.000672, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 87.578, "latency": 14.945, "stderr": 1.838, "cost_per_test": 0.000527, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 87.578, "latency": 27.485, "stderr": 1.838, "cost_per_test": 0.022583, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 86.025, "latency": 2.741, "stderr": 1.932, "cost_per_test": 0.00269, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-2-1212": { "accuracy": 85.714, "latency": 1.554, "stderr": 1.95, "cost_per_test": 0.002082, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/labs-devstral-small-2512": { "accuracy": 85.714, "latency": 3.373, "stderr": 1.95, "cost_per_test": 0.000106, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 85.714, "latency": 5.163, "stderr": 1.95, "cost_per_test": 0.000698, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 85.714, "latency": 8.556, "stderr": 1.95, "cost_per_test": 0.002431, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 85.404, "latency": 2.331, "stderr": 1.968, "cost_per_test": 0.000107, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 83.54, "latency": 2.421, "stderr": 2.066, "cost_per_test": 0.000567, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 83.54, "latency": 42.114, "stderr": 2.066, "cost_per_test": null, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 82.609, "latency": 1.061, "stderr": 2.112, "cost_per_test": 7.3e-05, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 81.677, "latency": 3.616, "stderr": 2.156, "cost_per_test": 0.002239, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 78.882, "latency": 20.386, "stderr": 2.275, "cost_per_test": 0.004396, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 73.913, "latency": 15.647, "stderr": 2.447, "cost_per_test": 0.065374, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 72.981, "latency": 3.225, "stderr": 2.475, "cost_per_test": 0.000104, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash-preview-04-17-thinking": { "accuracy": 68.634, "latency": 82.664, "stderr": 2.586, "cost_per_test": 0.000715, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 60.87, "latency": 6.066, "stderr": 2.72, "cost_per_test": 0.000237, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 57.453, "latency": 3.338, "stderr": 2.755, "cost_per_test": 0.002158, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "cohere/command-r-plus": { "accuracy": 48.447, "latency": 3.194, "stderr": 2.785, "cost_per_test": 0.002233, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 42.857, "latency": 3.336, "stderr": 2.758, "cost_per_test": 0.000263, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 27.64, "latency": 0.814, "stderr": 2.492, "cost_per_test": 0.000151, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "medium": { "openai/gpt-5-2025-08-07": { "accuracy": 92.689, "latency": 198.761, "stderr": 1.33, "cost_per_test": 0.06513, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 91.906, "latency": 20.205, "stderr": 1.394, "cost_per_test": 0.088064, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 91.645, "latency": 457.545, "stderr": 1.414, "cost_per_test": 0.074949, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 91.384, "latency": 18.737, "stderr": 1.434, "cost_per_test": 0.054248, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 91.123, "latency": 585.336, "stderr": 1.453, "cost_per_test": 0.082915, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 90.862, "latency": 61.049, "stderr": 1.472, "cost_per_test": 0.194043, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 90.601, "latency": 10.317, "stderr": 1.491, "cost_per_test": 0.028463, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 90.601, "latency": 20.162, "stderr": 1.491, "cost_per_test": 0.046729, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 90.601, "latency": 25.172, "stderr": 1.491, "cost_per_test": 0.044201, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 90.601, "latency": 35.468, "stderr": 1.491, "cost_per_test": 0.063422, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-codex": { "accuracy": 90.339, "latency": 85.052, "stderr": 1.51, "cost_per_test": 0.063351, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 90.339, "latency": 100.608, "stderr": 1.51, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 90.078, "latency": 241.235, "stderr": 1.528, "cost_per_test": 0.045918, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 89.817, "latency": 72.04, "stderr": 1.545, "cost_per_test": 0.083466, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 89.817, "latency": 82.371, "stderr": 1.545, "cost_per_test": 0.030294, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 89.817, "latency": 87.119, "stderr": 1.545, "cost_per_test": 0.162481, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 89.817, "latency": 155.532, "stderr": 1.545, "cost_per_test": 0.04141, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4-0709": { "accuracy": 89.817, "latency": 184.746, "stderr": 1.545, "cost_per_test": 0.122991, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 89.817, "latency": 206.616, "stderr": 1.545, "cost_per_test": 0.107396, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 89.817, "latency": 372.634, "stderr": 1.545, "cost_per_test": 0.050295, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 89.556, "latency": 93.286, "stderr": 1.563, "cost_per_test": 0.0899, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 89.556, "latency": 487.608, "stderr": 1.563, "cost_per_test": 0.029428, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 89.295, "latency": 23.607, "stderr": 1.58, "cost_per_test": 0.011845, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 89.295, "latency": 81.028, "stderr": 1.58, "cost_per_test": 0.024229, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 89.295, "latency": 88.206, "stderr": 1.58, "cost_per_test": 0.129689, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 89.295, "latency": 115.405, "stderr": 1.58, "cost_per_test": 0.049639, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 89.295, "latency": 124.73, "stderr": 1.58, "cost_per_test": 0.004362, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 89.295, "latency": 159.213, "stderr": 1.58, "cost_per_test": 0.182375, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 89.295, "latency": 252.124, "stderr": 1.58, "cost_per_test": 0.01724, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 89.295, "latency": 426.957, "stderr": 1.58, "cost_per_test": 0.00629, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 89.034, "latency": 25.656, "stderr": 1.597, "cost_per_test": 0.008427, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 89.034, "latency": 75.048, "stderr": 1.597, "cost_per_test": 0.02512, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 89.034, "latency": 82.766, "stderr": 1.597, "cost_per_test": 0.014546, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.1-codex": { "accuracy": 89.034, "latency": 117.498, "stderr": 1.597, "cost_per_test": 0.073947, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 89.034, "latency": 171.35, "stderr": 1.597, "cost_per_test": 0.003959, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.3": { "accuracy": 89.034, "latency": 1736.065, "stderr": 1.597, "cost_per_test": 0.013581, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 88.773, "latency": 199.444, "stderr": 1.613, "cost_per_test": 0.05386, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 88.512, "latency": 38.567, "stderr": 1.629, "cost_per_test": 0.001531, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 88.512, "latency": 97.385, "stderr": 1.629, "cost_per_test": 0.048182, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 88.512, "latency": 192.49, "stderr": 1.629, "cost_per_test": 0.008847, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 88.251, "latency": 146.467, "stderr": 1.645, "cost_per_test": 0.028865, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 88.251, "latency": 150.809, "stderr": 1.645, "cost_per_test": 0.937899, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 87.99, "latency": 92.758, "stderr": 1.661, "cost_per_test": 0.051088, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3": { "accuracy": 87.99, "latency": 157.495, "stderr": 1.661, "cost_per_test": 0.044316, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 87.99, "latency": 246.613, "stderr": 1.661, "cost_per_test": 0.039346, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 87.99, "latency": 308.563, "stderr": 1.661, "cost_per_test": 0.029814, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5.1": { "accuracy": 87.958, "latency": 201.438, "stderr": 1.665, "cost_per_test": 0.042412, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 87.728, "latency": 15.755, "stderr": 1.677, "cost_per_test": 0.011636, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.1-codex-max": { "accuracy": 87.728, "latency": 37.039, "stderr": 1.677, "cost_per_test": 0.00313, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 87.728, "latency": 56.706, "stderr": 1.677, "cost_per_test": 0.035761, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 87.728, "latency": 74.045, "stderr": 1.677, "cost_per_test": 0.051518, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 87.728, "latency": 98.163, "stderr": 1.677, "cost_per_test": 0.035956, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-codex": { "accuracy": 87.467, "latency": 73.006, "stderr": 1.692, "cost_per_test": 0.056783, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 87.467, "latency": 183.911, "stderr": 1.692, "cost_per_test": 0.018683, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-4.7": { "accuracy": 87.467, "latency": 256.071, "stderr": 1.692, "cost_per_test": 0.028885, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 87.206, "latency": 20.609, "stderr": 1.707, "cost_per_test": 0.001462, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 87.206, "latency": 85.426, "stderr": 1.707, "cost_per_test": 0.091357, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 87.206, "latency": 103.83, "stderr": 1.707, "cost_per_test": 0.03002, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 86.945, "latency": 100.091, "stderr": 1.722, "cost_per_test": 0.00197, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 86.945, "latency": 106.782, "stderr": 1.722, "cost_per_test": 0.005799, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 86.945, "latency": 165.683, "stderr": 1.722, "cost_per_test": 0.216735, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 86.684, "latency": 54.348, "stderr": 1.736, "cost_per_test": 0.006685, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-4.6": { "accuracy": 86.684, "latency": 136.313, "stderr": 1.736, "cost_per_test": 0.021343, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 86.423, "latency": 171.747, "stderr": 1.75, "cost_per_test": 0.013857, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 86.423, "latency": 207.281, "stderr": 1.75, "cost_per_test": 0.006168, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 86.162, "latency": 33.971, "stderr": 1.764, "cost_per_test": 0.004421, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-2.5-pro-preview-03-25": { "accuracy": 86.162, "latency": 168.41, "stderr": 1.764, "cost_per_test": 0.021947, "temperature": 1.0, "top_p": null, "max_output_tokens": 32512, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 85.901, "latency": 153.221, "stderr": 1.778, "cost_per_test": 0.007709, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 85.64, "latency": 27.834, "stderr": 1.792, "cost_per_test": 0.003139, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 85.64, "latency": 160.04, "stderr": 1.792, "cost_per_test": 0.212107, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 85.64, "latency": 248.827, "stderr": 1.792, "cost_per_test": 0.016778, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 85.117, "latency": 240.241, "stderr": 1.819, "cost_per_test": 0.002954, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 84.856, "latency": 18.406, "stderr": 1.832, "cost_per_test": 0.01721, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 84.856, "latency": 111.923, "stderr": 1.832, "cost_per_test": 0.020582, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 84.595, "latency": 83.197, "stderr": 1.845, "cost_per_test": 0.002696, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3-max": { "accuracy": 83.29, "latency": 81.531, "stderr": 1.906, "cost_per_test": 0.016419, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 82.507, "latency": 41.234, "stderr": 1.941, "cost_per_test": 0.002115, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 82.245, "latency": 18.454, "stderr": 1.953, "cost_per_test": 0.035584, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 82.245, "latency": 44.446, "stderr": 1.953, "cost_per_test": 0.002115, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 81.984, "latency": 34.18, "stderr": 1.964, "cost_per_test": 0.03838, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 81.723, "latency": 249.775, "stderr": 1.975, "cost_per_test": 0.025642, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 81.462, "latency": 13.904, "stderr": 1.986, "cost_per_test": 0.081319, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 80.679, "latency": 201.514, "stderr": 2.017, "cost_per_test": 0.048182, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 80.157, "latency": 590.013, "stderr": 2.038, "cost_per_test": 0.014575, "temperature": 1.0, "top_p": null, "max_output_tokens": 38912, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 79.634, "latency": 100.657, "stderr": 2.058, "cost_per_test": 0.015391, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 79.634, "latency": 115.126, "stderr": 2.058, "cost_per_test": 0.142184, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 78.59, "latency": 32.492, "stderr": 2.096, "cost_per_test": 0.000383, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-20250514-thinking": { "accuracy": 78.068, "latency": 95.662, "stderr": 2.114, "cost_per_test": 0.409474, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 77.285, "latency": 56.013, "stderr": 2.141, "cost_per_test": 0.006132, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 76.24, "latency": 81.165, "stderr": 2.175, "cost_per_test": 0.041335, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 75.979, "latency": 53.868, "stderr": 2.183, "cost_per_test": 0.03331, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 75.979, "latency": 237.196, "stderr": 2.183, "cost_per_test": 0.005591, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 74.413, "latency": 195.815, "stderr": 2.23, "cost_per_test": 0.00235, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 74.151, "latency": 34.315, "stderr": 2.237, "cost_per_test": 0.000597, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.2": { "accuracy": 74.151, "latency": 518.732, "stderr": 2.237, "cost_per_test": 0.133461, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-4.5": { "accuracy": 73.89, "latency": 487.993, "stderr": 2.244, "cost_per_test": 0.018933, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 72.324, "latency": 94.824, "stderr": 2.286, "cost_per_test": 0.376829, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 72.063, "latency": 25.669, "stderr": 2.293, "cost_per_test": 0.001743, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 71.802, "latency": 26.106, "stderr": 2.299, "cost_per_test": 0.002094, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 71.279, "latency": 20.827, "stderr": 2.312, "cost_per_test": 0.001617, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 71.018, "latency": 32.269, "stderr": 2.318, "cost_per_test": 0.055591, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 70.757, "latency": 14.302, "stderr": 2.324, "cost_per_test": 0.053676, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 69.713, "latency": 90.038, "stderr": 2.348, "cost_per_test": 0.012013, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 65.274, "latency": 44.943, "stderr": 2.433, "cost_per_test": 0.044958, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 65.013, "latency": 1260.379, "stderr": 2.437, "cost_per_test": 0.036469, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-code-fast-1": { "accuracy": 63.708, "latency": 18.484, "stderr": 2.457, "cost_per_test": 0.007001, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 60.574, "latency": 103.562, "stderr": 2.497, "cost_per_test": 0.135941, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 59.791, "latency": 151.89, "stderr": 2.505, "cost_per_test": null, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 59.269, "latency": 13.721, "stderr": 2.511, "cost_per_test": 0.013237, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 57.702, "latency": 17.92, "stderr": 2.524, "cost_per_test": 0.000936, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 56.658, "latency": 12.137, "stderr": 2.532, "cost_per_test": 0.008849, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 56.136, "latency": 9.553, "stderr": 2.536, "cost_per_test": 0.007034, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 54.83, "latency": 15.01, "stderr": 2.543, "cost_per_test": 0.01881, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 54.308, "latency": 87.77, "stderr": 2.545, "cost_per_test": 0.296634, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 53.525, "latency": 25.394, "stderr": 2.549, "cost_per_test": 0.001931, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "mistralai/devstral-2512": { "accuracy": 51.697, "latency": 83.438, "stderr": 2.553, "cost_per_test": 0.053806, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-3": { "accuracy": 51.175, "latency": 5.247, "stderr": 2.554, "cost_per_test": 0.009192, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-preview-04-17-thinking": { "accuracy": 45.692, "latency": 133.159, "stderr": 2.545, "cost_per_test": 0.001376, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 44.648, "latency": 11.374, "stderr": 2.54, "cost_per_test": 0.012137, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 43.864, "latency": 11.31, "stderr": 2.536, "cost_per_test": 0.000832, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 42.82, "latency": 13.49, "stderr": 2.528, "cost_per_test": 0.001719, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 41.514, "latency": 10.186, "stderr": 2.518, "cost_per_test": 0.000968, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 39.164, "latency": 4.643, "stderr": 2.494, "cost_per_test": 0.004063, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/labs-devstral-small-2512": { "accuracy": 35.248, "latency": 14.756, "stderr": 2.441, "cost_per_test": 0.000311, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 34.465, "latency": 10.747, "stderr": 2.428, "cost_per_test": 0.000199, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 32.898, "latency": 10.86, "stderr": 2.401, "cost_per_test": 0.003155, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 31.854, "latency": 3.388, "stderr": 2.381, "cost_per_test": 0.000308, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 31.332, "latency": 15.447, "stderr": 2.37, "cost_per_test": 0.000392, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 29.243, "latency": 3.998, "stderr": 2.324, "cost_per_test": 0.002077, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-2-1212": { "accuracy": 26.893, "latency": 2.387, "stderr": 2.266, "cost_per_test": 0.003005, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 25.587, "latency": 5.654, "stderr": 2.23, "cost_per_test": 0.003466, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 24.804, "latency": 86.787, "stderr": 2.207, "cost_per_test": 0.06398, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 23.76, "latency": 7.849, "stderr": 2.175, "cost_per_test": 0.006843, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 23.76, "latency": 19.756, "stderr": 2.175, "cost_per_test": 0.000625, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 22.977, "latency": 2.997, "stderr": 2.15, "cost_per_test": 0.000168, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 21.671, "latency": 44.488, "stderr": 2.105, "cost_per_test": null, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 20.627, "latency": 2.835, "stderr": 2.068, "cost_per_test": 0.000732, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 19.321, "latency": 3.485, "stderr": 2.017, "cost_per_test": 0.000152, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 16.971, "latency": 7.076, "stderr": 1.918, "cost_per_test": 0.000295, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 8.094, "latency": 5.106, "stderr": 1.394, "cost_per_test": 0.003471, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "cohere/command-r-plus": { "accuracy": 6.266, "latency": 6.612, "stderr": 1.238, "cost_per_test": 0.003262, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 3.916, "latency": 5.684, "stderr": 0.991, "cost_per_test": 0.000414, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 1.828, "latency": 1.304, "stderr": 0.684, "cost_per_test": 0.000207, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null } }, "hard": { "anthropic/claude-fable-5-1": { "accuracy": 80.571, "latency": 154.12, "stderr": 2.115, "cost_per_test": 0.678233, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 79.714, "latency": 281.848, "stderr": 2.149, "cost_per_test": 1.056984, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 78.0, "latency": 49.71, "stderr": 2.214, "cost_per_test": 0.14965, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 77.429, "latency": 153.772, "stderr": 2.235, "cost_per_test": 0.348014, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 76.571, "latency": 175.343, "stderr": 2.264, "cost_per_test": 0.198871, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 76.571, "latency": 266.652, "stderr": 2.264, "cost_per_test": 0.089228, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 76.286, "latency": 30.07, "stderr": 2.273, "cost_per_test": 0.084226, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 75.714, "latency": 791.554, "stderr": 2.292, "cost_per_test": 0.213498, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.3-codex": { "accuracy": 75.143, "latency": 306.379, "stderr": 2.31, "cost_per_test": 0.209533, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 74.571, "latency": 63.562, "stderr": 2.328, "cost_per_test": 0.11252, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 74.571, "latency": 379.558, "stderr": 2.328, "cost_per_test": 0.798518, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.2-codex": { "accuracy": 74.571, "latency": 409.148, "stderr": 2.328, "cost_per_test": 0.279114, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 74.286, "latency": 486.089, "stderr": 2.336, "cost_per_test": 0.255825, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 74.0, "latency": 286.143, "stderr": 2.345, "cost_per_test": 0.044236, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 74.0, "latency": 645.626, "stderr": 2.345, "cost_per_test": 0.14552, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 73.429, "latency": 439.27, "stderr": 2.361, "cost_per_test": 0.20172, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 73.429, "latency": 831.288, "stderr": 2.361, "cost_per_test": 0.100417, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 73.143, "latency": 144.935, "stderr": 2.369, "cost_per_test": 0.177857, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 72.857, "latency": 113.134, "stderr": 2.377, "cost_per_test": 0.079222, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 72.571, "latency": 1487.685, "stderr": 2.385, "cost_per_test": 0.23531, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5-mini-2025-08-07": { "accuracy": 71.714, "latency": 68.622, "stderr": 2.407, "cost_per_test": 0.024106, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 71.429, "latency": 1138.039, "stderr": 2.415, "cost_per_test": 0.137579, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.1-2025-11-13": { "accuracy": 71.143, "latency": 311.987, "stderr": 2.422, "cost_per_test": 0.273252, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-pro-preview": { "accuracy": 70.857, "latency": 131.816, "stderr": 2.429, "cost_per_test": 0.19064, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.2-2025-12-11": { "accuracy": 70.118, "latency": 354.881, "stderr": 2.49, "cost_per_test": 0.30169, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 69.714, "latency": 136.003, "stderr": 2.456, "cost_per_test": 0.074818, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 69.714, "latency": 353.98, "stderr": 2.456, "cost_per_test": 0.151373, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 69.429, "latency": 624.042, "stderr": 2.463, "cost_per_test": 0.043388, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 69.143, "latency": 238.402, "stderr": 2.469, "cost_per_test": 0.100507, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 69.143, "latency": 366.032, "stderr": 2.469, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-5.1-codex": { "accuracy": 68.857, "latency": 569.836, "stderr": 2.475, "cost_per_test": 0.331205, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5-codex": { "accuracy": 68.571, "latency": 323.06, "stderr": 2.481, "cost_per_test": 0.228984, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 68.286, "latency": 550.482, "stderr": 2.487, "cost_per_test": 0.341104, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 67.714, "latency": 421.687, "stderr": 2.499, "cost_per_test": 0.743287, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 67.429, "latency": 146.86, "stderr": 2.505, "cost_per_test": 0.226187, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.5-plus-thinking": { "accuracy": 67.143, "latency": 355.842, "stderr": 2.511, "cost_per_test": 0.07058, "temperature": 0.6, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 67.143, "latency": 764.464, "stderr": 2.511, "cost_per_test": 0.124575, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 66.857, "latency": 180.069, "stderr": 2.516, "cost_per_test": 0.223535, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 66.857, "latency": 787.584, "stderr": 2.516, "cost_per_test": 1.012836, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5-2025-08-07": { "accuracy": 66.286, "latency": 225.456, "stderr": 2.527, "cost_per_test": 0.107337, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-6-thinking": { "accuracy": 66.286, "latency": 798.957, "stderr": 2.527, "cost_per_test": 0.829373, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 66.0, "latency": 132.769, "stderr": 2.532, "cost_per_test": 0.00563, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 66.0, "latency": 149.086, "stderr": 2.532, "cost_per_test": 0.113578, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 66.0, "latency": 157.018, "stderr": 2.532, "cost_per_test": 0.007351, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.3": { "accuracy": 66.0, "latency": 6285.07, "stderr": 2.532, "cost_per_test": 0.04454, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 65.714, "latency": 331.047, "stderr": 2.537, "cost_per_test": 0.101245, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.1-codex-max": { "accuracy": 65.429, "latency": 151.6, "stderr": 2.542, "cost_per_test": -0.000378, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/gpt-oss-120b": { "accuracy": 64.571, "latency": 150.782, "stderr": 2.557, "cost_per_test": 0.017407, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 64.286, "latency": 415.345, "stderr": 2.561, "cost_per_test": 0.079055, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/o3-2025-04-16": { "accuracy": 64.0, "latency": 100.796, "stderr": 2.566, "cost_per_test": 0.068741, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-5-20251101-thinking": { "accuracy": 64.0, "latency": 302.373, "stderr": 2.566, "cost_per_test": 1.713524, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.5-flash": { "accuracy": 62.571, "latency": 348.754, "stderr": 2.587, "cost_per_test": 0.016007, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.1": { "accuracy": 62.571, "latency": 529.6, "stderr": 2.587, "cost_per_test": 0.038707, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 61.714, "latency": 442.662, "stderr": 2.598, "cost_per_test": 0.076308, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-4.7": { "accuracy": 61.098, "latency": 884.647, "stderr": 2.332, "cost_per_test": 0.073162, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 60.857, "latency": 388.241, "stderr": 2.609, "cost_per_test": 0.060639, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4-0709": { "accuracy": 60.857, "latency": 452.981, "stderr": 2.609, "cost_per_test": 0.312336, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 60.571, "latency": 703.081, "stderr": 2.612, "cost_per_test": 0.872008, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 59.714, "latency": 723.191, "stderr": 2.622, "cost_per_test": 0.076845, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 59.143, "latency": 191.375, "stderr": 2.628, "cost_per_test": 0.004286, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/deepseek-v3p2-thinking": { "accuracy": 59.143, "latency": 474.722, "stderr": 2.628, "cost_per_test": 0.006164, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5-thinking": { "accuracy": 58.857, "latency": 565.964, "stderr": 2.63, "cost_per_test": 0.084257, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/o4-mini-2025-04-16": { "accuracy": 58.571, "latency": 67.813, "stderr": 2.633, "cost_per_test": 0.02575, "temperature": null, "top_p": null, "max_output_tokens": 100000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4-1-fast-reasoning": { "accuracy": 58.571, "latency": 209.035, "stderr": 2.633, "cost_per_test": 0.008272, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 58.571, "latency": 562.035, "stderr": 2.633, "cost_per_test": 0.01277, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-4.6": { "accuracy": 58.286, "latency": 546.351, "stderr": 2.636, "cost_per_test": 0.055417, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 122880, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.1": { "accuracy": 57.429, "latency": 433.76, "stderr": 2.643, "cost_per_test": 0.086606, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.5": { "accuracy": 57.143, "latency": 642.154, "stderr": 2.645, "cost_per_test": 0.068525, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 196608, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "fireworks/gpt-oss-20b": { "accuracy": 56.389, "latency": 205.668, "stderr": 2.614, "cost_per_test": 0.013366, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 56.0, "latency": 664.796, "stderr": 2.653, "cost_per_test": 0.044094, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 55.429, "latency": 43.676, "stderr": 2.657, "cost_per_test": 0.006597, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3": { "accuracy": 54.857, "latency": 564.831, "stderr": 2.66, "cost_per_test": 0.158062, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 53.429, "latency": 39.678, "stderr": 2.666, "cost_per_test": 0.035699, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 53.429, "latency": 717.881, "stderr": 2.666, "cost_per_test": 0.016865, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4-fast-reasoning": { "accuracy": 53.143, "latency": 119.467, "stderr": 2.667, "cost_per_test": 0.009458, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-pro-preview-03-25": { "accuracy": 52.571, "latency": 228.861, "stderr": 2.669, "cost_per_test": 0.02524, "temperature": 1.0, "top_p": null, "max_output_tokens": 32512, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3-max": { "accuracy": 52.286, "latency": 217.563, "stderr": 2.67, "cost_per_test": 0.042105, "temperature": 0.7, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-2.5-flash-preview-09-2025-thinking": { "accuracy": 48.571, "latency": 83.508, "stderr": 2.672, "cost_per_test": 0.003962, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/magistral-medium-2509": { "accuracy": 45.143, "latency": 374.337, "stderr": 2.66, "cost_per_test": 0.081503, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash-preview-09-2025": { "accuracy": 44.857, "latency": 84.82, "stderr": 2.658, "cost_per_test": 0.003876, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-5-20251101": { "accuracy": 44.571, "latency": 22.051, "stderr": 2.657, "cost_per_test": 0.124578, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "high", "provider": "Anthropic", "harness": null }, "grok/grok-3-mini-fast-high-reasoning": { "accuracy": 44.0, "latency": 457.186, "stderr": 2.653, "cost_per_test": 0.013734, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-5-20250929-thinking": { "accuracy": 40.286, "latency": 174.048, "stderr": 2.622, "cost_per_test": 0.213186, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o3-mini-2025-01-31": { "accuracy": 39.714, "latency": 90.45, "stderr": 2.615, "cost_per_test": 0.057867, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/deepseek-v3p2": { "accuracy": 39.143, "latency": 76.711, "stderr": 2.609, "cost_per_test": 0.001234, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/magistral-small-2509": { "accuracy": 38.0, "latency": 172.409, "stderr": 2.594, "cost_per_test": 0.024321, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025-thinking": { "accuracy": 37.429, "latency": 72.684, "stderr": 2.587, "cost_per_test": 0.000652, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.2": { "accuracy": 37.143, "latency": 1020.063, "stderr": 2.583, "cost_per_test": 0.239479, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "together/moonshotai/Kimi-K2-Instruct": { "accuracy": 36.857, "latency": 126.698, "stderr": 2.579, "cost_per_test": 0.012321, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "fireworks/deepseek-r1": { "accuracy": 36.286, "latency": 150.553, "stderr": 2.57, "cost_per_test": 0.067513, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/qwen3-235b-a22b": { "accuracy": 35.429, "latency": 556.283, "stderr": 2.557, "cost_per_test": 0.01294, "temperature": 1.0, "top_p": null, "max_output_tokens": 38912, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-2.5-flash-lite-preview-09-2025": { "accuracy": 33.429, "latency": 23.363, "stderr": 2.522, "cost_per_test": 0.003753, "temperature": 1.0, "top_p": null, "max_output_tokens": 65535, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-20250514-thinking": { "accuracy": 33.429, "latency": 145.059, "stderr": 2.522, "cost_per_test": 0.609114, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3-max-preview": { "accuracy": 32.571, "latency": 172.001, "stderr": 2.505, "cost_per_test": 0.025552, "temperature": 0.7, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 30.857, "latency": 292.67, "stderr": 2.469, "cost_per_test": 0.004002, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-m.1": { "accuracy": 30.857, "latency": 340.727, "stderr": 2.469, "cost_per_test": 0.009652, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "zai/glm-4.5": { "accuracy": 30.0, "latency": 551.224, "stderr": 2.449, "cost_per_test": 0.029016, "temperature": 0.6, "top_p": 1.0, "max_output_tokens": 81920, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-3-mini-fast-low-reasoning": { "accuracy": 28.857, "latency": 56.247, "stderr": 2.422, "cost_per_test": 0.003407, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": "low", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/deepseek-v3-0324": { "accuracy": 28.571, "latency": 46.773, "stderr": 2.415, "cost_per_test": 0.002913, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-4-1-20250805-thinking": { "accuracy": 28.286, "latency": 143.594, "stderr": 2.407, "cost_per_test": 0.550873, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-code-fast-1": { "accuracy": 26.857, "latency": 53.943, "stderr": 2.369, "cost_per_test": 0.016003, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 40000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-2.5-flash-preview-04-17-thinking": { "accuracy": 26.286, "latency": 159.017, "stderr": 2.353, "cost_per_test": 0.00387, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2-thinking": { "accuracy": 26.286, "latency": 1894.45, "stderr": 2.353, "cost_per_test": 0.055589, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-1-20250805": { "accuracy": 25.143, "latency": 42.675, "stderr": 2.319, "cost_per_test": 0.074597, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514-thinking": { "accuracy": 23.143, "latency": 56.29, "stderr": 2.254, "cost_per_test": 0.054828, "temperature": null, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-20250514": { "accuracy": 22.857, "latency": 19.854, "stderr": 2.245, "cost_per_test": 0.019844, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/o1-2024-12-17": { "accuracy": 22.571, "latency": 172.829, "stderr": 2.235, "cost_per_test": 0.622193, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-3-7-sonnet-20250219-thinking": { "accuracy": 22.286, "latency": 128.992, "stderr": 2.224, "cost_per_test": 0.166043, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5-nano-2025-08-07": { "accuracy": 22.286, "latency": 271.445, "stderr": 2.224, "cost_per_test": 0.00966, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": "medium", "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4.1-mini-2025-04-14": { "accuracy": 21.429, "latency": 62.607, "stderr": 2.193, "cost_per_test": 0.002221, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-20250514": { "accuracy": 20.857, "latency": 20.172, "stderr": 2.172, "cost_per_test": 0.077502, "temperature": 1.0, "top_p": null, "max_output_tokens": 32000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561-thinking": { "accuracy": 20.286, "latency": 232.563, "stderr": 2.149, "cost_per_test": null, "temperature": 0.6, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-2.5-flash-preview-04-17": { "accuracy": 19.429, "latency": 35.389, "stderr": 2.115, "cost_per_test": 0.025301, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-3-7-sonnet-20250219": { "accuracy": 18.571, "latency": 27.525, "stderr": 2.079, "cost_per_test": 0.03584, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-2512": { "accuracy": 17.143, "latency": 53.158, "stderr": 2.015, "cost_per_test": 0.003963, "temperature": 0.065, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-3": { "accuracy": 13.429, "latency": 8.513, "stderr": 1.823, "cost_per_test": 0.013436, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4.1-2025-04-14": { "accuracy": 13.143, "latency": 79.95, "stderr": 1.806, "cost_per_test": 0.018997, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/devstral-2512": { "accuracy": 13.143, "latency": 159.536, "stderr": 1.806, "cost_per_test": 0.111846, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-3-5-sonnet-20241022": { "accuracy": 12.0, "latency": 12.952, "stderr": 1.737, "cost_per_test": 0.013989, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 11.143, "latency": 130.412, "stderr": 1.682, "cost_per_test": 0.092739, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/llama4-maverick-instruct-basic": { "accuracy": 10.0, "latency": 10.527, "stderr": 1.604, "cost_per_test": 0.000991, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4-fast-non-reasoning": { "accuracy": 8.571, "latency": 14.244, "stderr": 1.496, "cost_per_test": 0.001507, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/labs-devstral-small-2512": { "accuracy": 8.571, "latency": 38.334, "stderr": 1.496, "cost_per_test": 0.000622, "temperature": 0.2, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-2.0-flash-001": { "accuracy": 8.286, "latency": 5.372, "stderr": 1.473, "cost_per_test": 0.000482, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4.1-nano-2025-04-14": { "accuracy": 8.286, "latency": 22.324, "stderr": 1.473, "cost_per_test": 0.000588, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-1.5-pro-002": { "accuracy": 7.714, "latency": 5.667, "stderr": 1.426, "cost_per_test": 0.002659, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4-1-fast-non-reasoning": { "accuracy": 7.714, "latency": 17.265, "stderr": 1.426, "cost_per_test": 0.000716, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-3-5-haiku-20241022": { "accuracy": 7.143, "latency": 12.326, "stderr": 1.377, "cost_per_test": 0.003605, "temperature": 1.0, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-2505": { "accuracy": 6.0, "latency": 18.796, "stderr": 1.269, "cost_per_test": 0.002413, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-1.5-flash-002": { "accuracy": 5.143, "latency": 3.478, "stderr": 1.181, "cost_per_test": 0.0002, "temperature": null, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-4o-2024-11-20": { "accuracy": 5.143, "latency": 5.656, "stderr": 1.181, "cost_per_test": 0.005244, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "accuracy": 4.857, "latency": 3.58, "stderr": 1.149, "cost_per_test": 0.000931, "temperature": null, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "accuracy": 4.286, "latency": 15.586, "stderr": 1.083, "cost_per_test": 0.000694, "temperature": null, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "mistralai/mistral-large-2411": { "accuracy": 4.0, "latency": 6.776, "stderr": 1.047, "cost_per_test": 0.004168, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "together/langston/nim/nvidia/llama-3.3-nemotron-super-49b-v1-42e84561": { "accuracy": 3.714, "latency": 46.083, "stderr": 1.011, "cost_per_test": null, "temperature": 0.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-2-1212": { "accuracy": 3.429, "latency": 3.192, "stderr": 0.973, "cost_per_test": 0.003925, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 0, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-small-2503": { "accuracy": 3.143, "latency": 5.311, "stderr": 0.933, "cost_per_test": 0.000186, "temperature": 0.7, "top_p": null, "max_output_tokens": 8192, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "cohere/command-a-03-2025": { "accuracy": 2.571, "latency": 7.003, "stderr": 0.846, "cost_per_test": 0.008464, "temperature": 0.3, "top_p": null, "max_output_tokens": 8000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "ai21labs/jamba-large-1.6": { "accuracy": 1.429, "latency": 6.075, "stderr": 0.634, "cost_per_test": 0.004078, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "openai/gpt-4o-mini-2024-07-18": { "accuracy": 1.429, "latency": 8.949, "stderr": 0.634, "cost_per_test": 0.000362, "temperature": 1.0, "top_p": null, "max_output_tokens": 16384, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-small-2402": { "accuracy": 0.571, "latency": 6.101, "stderr": 0.403, "cost_per_test": 0.00047, "temperature": 0.7, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ai21labs/jamba-mini-1.6": { "accuracy": 0.286, "latency": 1.298, "stderr": 0.285, "cost_per_test": 0.000239, "temperature": 0.4, "top_p": null, "max_output_tokens": 4096, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "AI21 Labs", "harness": null }, "cohere/command-r-plus": { "accuracy": 0.0, "latency": 6.583, "stderr": 0.0, "cost_per_test": 0.004306, "temperature": 0.3, "top_p": null, "max_output_tokens": 4000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null } } } }