{ "metadata": { "benchmark": "ProgramBench", "slug": "programbench", "description": "Can language models rebuild programs from scratch?", "benchmark_id": "programbench", "family": "programbench", "version": "1", "updated": "2026-10-12", "dataset_type": "public", "industry": "coding", "tasks": { "overall": "Overall", "strict": "Fully Resolved", "almost": "Almost Resolved", "partial": "Raw Pass Rate" }, "models": [ "alibaba/qwen3.6-plus", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "fireworks/nemotron-lightning-3p5-30b-a3b", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-4.7", "kimi/kimi-k2.6", "kimi/kimi-k2.7-code", "kimi/kimi-k3", "meta/muse-wonder-full", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "minimax/MiniMax-M2.7", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-2026-03-05-high", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "poolside/laguna-m.1", "poolside/laguna-xs.2", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.1", "zai/glm-5.2", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": true, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 64 }, "tasks": { "overall": { "anthropic/claude-opus-5-5": { "accuracy": 18.5, "tie_breaker_score": 65.009, "latency": 8660.211, "stderr": 2.753, "cost_per_test": 68.051144, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 7.0, "tie_breaker_score": 53.508, "latency": 8791.194, "stderr": 1.809, "cost_per_test": 58.51711, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 6.5, "tie_breaker_score": 49.508, "latency": 7602.078, "stderr": 1.748, "cost_per_test": 35.860741, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 5.5, "tie_breaker_score": 50.009, "latency": 1621.514, "stderr": 1.616, "cost_per_test": 11.574605, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 3.0, "tie_breaker_score": 46.009, "latency": 4223.187, "stderr": 1.209, "cost_per_test": 2.868128, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 3.0, "tie_breaker_score": 41.508, "latency": 10480.572, "stderr": 1.209, "cost_per_test": 60.289268, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 2.5, "tie_breaker_score": 41.008, "latency": 4252.235, "stderr": 1.107, "cost_per_test": 17.656424, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 2.5, "tie_breaker_score": 29.508, "latency": 4389.65, "stderr": 1.107, "cost_per_test": 6.828942, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 2.5, "tie_breaker_score": 19.007, "latency": 19703.41, "stderr": 1.107, "cost_per_test": 54.285113, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 2.0, "tie_breaker_score": 33.008, "latency": 9447.46, "stderr": 0.992, "cost_per_test": 75.675797, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 2.0, "tie_breaker_score": 32.508, "latency": 2771.488, "stderr": 0.992, "cost_per_test": 11.672651, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 2.0, "tie_breaker_score": 17.506, "latency": 20494.929, "stderr": 0.992, "cost_per_test": 70.475273, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 1.5, "tie_breaker_score": 23.008, "latency": 1764.488, "stderr": 0.862, "cost_per_test": 15.289359, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 1.5, "tie_breaker_score": 16.507, "latency": 15354.08, "stderr": 0.862, "cost_per_test": 21.957364, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.6": { "accuracy": 1.0, "tie_breaker_score": 19.007, "latency": 16044.462, "stderr": 0.705, "cost_per_test": 16.759019, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 1.0, "tie_breaker_score": 15.507, "latency": 5523.099, "stderr": 0.705, "cost_per_test": 31.267424, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 1.0, "tie_breaker_score": 14.007, "latency": 2799.268, "stderr": 0.705, "cost_per_test": 10.928509, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 0.5, "tie_breaker_score": 21.508, "latency": 13078.924, "stderr": 0.5, "cost_per_test": 46.490946, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.5, "tie_breaker_score": 17.507, "latency": 12091.441, "stderr": 0.5, "cost_per_test": 0.470526, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.5, "tie_breaker_score": 16.507, "latency": 3709.707, "stderr": 0.5, "cost_per_test": 0.896935, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.5, "tie_breaker_score": 15.007, "latency": 2016.05, "stderr": 0.5, "cost_per_test": 5.377882, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 0.5, "tie_breaker_score": 13.007, "latency": 4112.719, "stderr": 0.5, "cost_per_test": 11.555338, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.5, "tie_breaker_score": 12.507, "latency": 10825.24, "stderr": 0.5, "cost_per_test": 0.726582, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.5, "tie_breaker_score": 9.507, "latency": 1322.959, "stderr": 0.5, "cost_per_test": 6.95454, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 0.5, "tie_breaker_score": 9.506, "latency": 7588.932, "stderr": 0.5, "cost_per_test": 12.57696, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.5, "tie_breaker_score": 8.007, "latency": 1734.863, "stderr": 0.5, "cost_per_test": 0.180944, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 0.5, "tie_breaker_score": 8.002, "latency": 21822.228, "stderr": 0.5, "cost_per_test": 28.645593, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.4-2026-03-05-high": { "accuracy": 0.5, "tie_breaker_score": 3.506, "latency": 958.495, "stderr": 0.5, "cost_per_test": 2.27429, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 0.5, "tie_breaker_score": 2.506, "latency": 1598.124, "stderr": 0.5, "cost_per_test": 2.467822, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.5, "tie_breaker_score": 2.006, "latency": 4349.841, "stderr": 0.5, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.5, "tie_breaker_score": 0.504, "latency": 1710.815, "stderr": 0.5, "cost_per_test": 0.479317, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "tie_breaker_score": 14.507, "latency": 7801.049, "stderr": 0.0, "cost_per_test": 12.809118, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "tie_breaker_score": 13.507, "latency": 4970.276, "stderr": 0.0, "cost_per_test": 0.431698, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "tie_breaker_score": 13.507, "latency": 5471.639, "stderr": 0.0, "cost_per_test": 24.358317, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "tie_breaker_score": 10.504, "latency": 21263.689, "stderr": 0.0, "cost_per_test": 12.357107, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "tie_breaker_score": 9.507, "latency": 1290.211, "stderr": 0.0, "cost_per_test": 7.130366, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "tie_breaker_score": 9.506, "latency": 3246.698, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "tie_breaker_score": 9.007, "latency": 2493.897, "stderr": 0.0, "cost_per_test": 0.942725, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "tie_breaker_score": 8.506, "latency": 20327.596, "stderr": 0.0, "cost_per_test": 1.285061, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "tie_breaker_score": 7.007, "latency": 2386.411, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "tie_breaker_score": 7.005, "latency": 19801.258, "stderr": 0.0, "cost_per_test": 0.79862, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "tie_breaker_score": 4.507, "latency": 1139.043, "stderr": 0.0, "cost_per_test": 4.124197, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "tie_breaker_score": 3.005, "latency": 3225.344, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "tie_breaker_score": 2.506, "latency": 2852.749, "stderr": 0.0, "cost_per_test": 5.923527, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 0.0, "tie_breaker_score": 2.503, "latency": 20592.047, "stderr": 0.0, "cost_per_test": 13.063498, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "tie_breaker_score": 1.506, "latency": 1104.517, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "tie_breaker_score": 1.005, "latency": 3931.735, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "tie_breaker_score": 1.004, "latency": 755.928, "stderr": 0.0, "cost_per_test": 2.629095, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "tie_breaker_score": 1.003, "latency": 1981.11, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "tie_breaker_score": 1.001, "latency": 21400.403, "stderr": 0.0, "cost_per_test": 17.990872, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 0.0, "tie_breaker_score": 0.505, "latency": 3860.332, "stderr": 0.0, "cost_per_test": 3.185154, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "tie_breaker_score": 0.505, "latency": 4794.524, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "tie_breaker_score": 0.501, "latency": 3718.555, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "tie_breaker_score": 0.005, "latency": 3778.098, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "tie_breaker_score": 0.003, "latency": 418.92, "stderr": 0.0, "cost_per_test": 0.439965, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "tie_breaker_score": 0.003, "latency": 434.999, "stderr": 0.0, "cost_per_test": 0.337485, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "tie_breaker_score": 0.003, "latency": 457.113, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "tie_breaker_score": 0.002, "latency": 236.134, "stderr": 0.0, "cost_per_test": 0.355823, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "tie_breaker_score": 0.002, "latency": 1465.949, "stderr": 0.0, "cost_per_test": 0.216901, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "tie_breaker_score": 0.002, "latency": 1541.675, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "tie_breaker_score": 0.002, "latency": 2734.146, "stderr": 0.0, "cost_per_test": 4.143489, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "tie_breaker_score": 0.002, "latency": 4921.898, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "tie_breaker_score": 0.001, "latency": 176.246, "stderr": 0.0, "cost_per_test": 0.063906, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "tie_breaker_score": 0.001, "latency": 1460.176, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null } }, "strict": { "anthropic/claude-opus-5-5": { "accuracy": 18.5, "tie_breaker_score": 65.009, "latency": 8660.211, "stderr": 2.753, "cost_per_test": 68.051144, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 7.0, "tie_breaker_score": 53.508, "latency": 8791.194, "stderr": 1.809, "cost_per_test": 58.51711, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 6.5, "tie_breaker_score": 49.508, "latency": 7602.078, "stderr": 1.748, "cost_per_test": 35.860741, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 5.5, "tie_breaker_score": 50.009, "latency": 1621.514, "stderr": 1.616, "cost_per_test": 11.574605, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 3.0, "tie_breaker_score": 46.009, "latency": 4223.187, "stderr": 1.209, "cost_per_test": 2.868128, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 3.0, "tie_breaker_score": 41.508, "latency": 10480.572, "stderr": 1.209, "cost_per_test": 60.289268, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 2.5, "tie_breaker_score": 41.008, "latency": 4252.235, "stderr": 1.107, "cost_per_test": 17.656424, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 2.5, "tie_breaker_score": 29.508, "latency": 4389.65, "stderr": 1.107, "cost_per_test": 6.828942, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 2.5, "tie_breaker_score": 19.007, "latency": 19703.41, "stderr": 1.107, "cost_per_test": 54.285113, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 2.0, "tie_breaker_score": 33.008, "latency": 9447.46, "stderr": 0.992, "cost_per_test": 75.675797, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 2.0, "tie_breaker_score": 32.508, "latency": 2771.488, "stderr": 0.992, "cost_per_test": 11.672651, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 2.0, "tie_breaker_score": 17.506, "latency": 20494.929, "stderr": 0.992, "cost_per_test": 70.475273, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 1.5, "tie_breaker_score": 23.008, "latency": 1764.488, "stderr": 0.862, "cost_per_test": 15.289359, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 1.5, "tie_breaker_score": 16.507, "latency": 15354.08, "stderr": 0.862, "cost_per_test": 21.957364, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.6": { "accuracy": 1.0, "tie_breaker_score": 19.007, "latency": 16044.462, "stderr": 0.705, "cost_per_test": 16.759019, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 1.0, "tie_breaker_score": 15.507, "latency": 5523.099, "stderr": 0.705, "cost_per_test": 31.267424, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 1.0, "tie_breaker_score": 14.007, "latency": 2799.268, "stderr": 0.705, "cost_per_test": 10.928509, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 0.5, "tie_breaker_score": 21.508, "latency": 13078.924, "stderr": 0.5, "cost_per_test": 46.490946, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.5, "tie_breaker_score": 17.507, "latency": 12091.441, "stderr": 0.5, "cost_per_test": 0.470526, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.5, "tie_breaker_score": 16.507, "latency": 3709.707, "stderr": 0.5, "cost_per_test": 0.896935, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.5, "tie_breaker_score": 15.007, "latency": 2016.05, "stderr": 0.5, "cost_per_test": 5.377882, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 0.5, "tie_breaker_score": 13.007, "latency": 4112.719, "stderr": 0.5, "cost_per_test": 11.555338, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.5, "tie_breaker_score": 12.507, "latency": 10825.24, "stderr": 0.5, "cost_per_test": 0.726582, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.5, "tie_breaker_score": 9.507, "latency": 1322.959, "stderr": 0.5, "cost_per_test": 6.95454, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 0.5, "tie_breaker_score": 9.506, "latency": 7588.932, "stderr": 0.5, "cost_per_test": 12.57696, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.5, "tie_breaker_score": 8.007, "latency": 1734.863, "stderr": 0.5, "cost_per_test": 0.180944, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 0.5, "tie_breaker_score": 8.002, "latency": 21822.228, "stderr": 0.5, "cost_per_test": 28.645593, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.4-2026-03-05-high": { "accuracy": 0.5, "tie_breaker_score": 3.506, "latency": 958.495, "stderr": 0.5, "cost_per_test": 2.27429, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 0.5, "tie_breaker_score": 2.506, "latency": 1598.124, "stderr": 0.5, "cost_per_test": 2.467822, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.5, "tie_breaker_score": 2.006, "latency": 4349.841, "stderr": 0.5, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.5, "tie_breaker_score": 0.504, "latency": 1710.815, "stderr": 0.5, "cost_per_test": 0.479317, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "tie_breaker_score": 14.507, "latency": 7801.049, "stderr": 0.0, "cost_per_test": 12.809118, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "tie_breaker_score": 13.507, "latency": 4970.276, "stderr": 0.0, "cost_per_test": 0.431698, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "tie_breaker_score": 13.507, "latency": 5471.639, "stderr": 0.0, "cost_per_test": 24.358317, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "tie_breaker_score": 10.504, "latency": 21263.689, "stderr": 0.0, "cost_per_test": 12.357107, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "tie_breaker_score": 9.507, "latency": 1290.211, "stderr": 0.0, "cost_per_test": 7.130366, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "tie_breaker_score": 9.506, "latency": 3246.698, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "tie_breaker_score": 9.007, "latency": 2493.897, "stderr": 0.0, "cost_per_test": 0.942725, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "tie_breaker_score": 8.506, "latency": 20327.596, "stderr": 0.0, "cost_per_test": 1.285061, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "tie_breaker_score": 7.007, "latency": 2386.411, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "tie_breaker_score": 7.005, "latency": 19801.258, "stderr": 0.0, "cost_per_test": 0.79862, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "tie_breaker_score": 4.507, "latency": 1139.043, "stderr": 0.0, "cost_per_test": 4.124197, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "tie_breaker_score": 3.005, "latency": 3225.344, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "tie_breaker_score": 2.506, "latency": 2852.749, "stderr": 0.0, "cost_per_test": 5.923527, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 0.0, "tie_breaker_score": 2.503, "latency": 20592.047, "stderr": 0.0, "cost_per_test": 13.063498, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "tie_breaker_score": 1.506, "latency": 1104.517, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "tie_breaker_score": 1.005, "latency": 3931.735, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "tie_breaker_score": 1.004, "latency": 755.928, "stderr": 0.0, "cost_per_test": 2.629095, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "tie_breaker_score": 1.003, "latency": 1981.11, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "tie_breaker_score": 1.001, "latency": 21400.403, "stderr": 0.0, "cost_per_test": 17.990872, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 0.0, "tie_breaker_score": 0.505, "latency": 3860.332, "stderr": 0.0, "cost_per_test": 3.185154, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "tie_breaker_score": 0.505, "latency": 4794.524, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "tie_breaker_score": 0.501, "latency": 3718.555, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "tie_breaker_score": 0.005, "latency": 3778.098, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "tie_breaker_score": 0.003, "latency": 418.92, "stderr": 0.0, "cost_per_test": 0.439965, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "tie_breaker_score": 0.003, "latency": 434.999, "stderr": 0.0, "cost_per_test": 0.337485, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "tie_breaker_score": 0.003, "latency": 457.113, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "tie_breaker_score": 0.002, "latency": 236.134, "stderr": 0.0, "cost_per_test": 0.355823, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "tie_breaker_score": 0.002, "latency": 1465.949, "stderr": 0.0, "cost_per_test": 0.216901, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "tie_breaker_score": 0.002, "latency": 1541.675, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "tie_breaker_score": 0.002, "latency": 2734.146, "stderr": 0.0, "cost_per_test": 4.143489, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "tie_breaker_score": 0.002, "latency": 4921.898, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "tie_breaker_score": 0.001, "latency": 176.246, "stderr": 0.0, "cost_per_test": 0.063906, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "tie_breaker_score": 0.001, "latency": 1460.176, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null } }, "partial": { "anthropic/claude-opus-5-5": { "accuracy": 86.986, "latency": 8660.211, "stderr": 1.792, "cost_per_test": 68.051144, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 85.417, "latency": 1621.514, "stderr": 1.6, "cost_per_test": 11.574605, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 85.25, "latency": 4223.187, "stderr": 1.517, "cost_per_test": 2.868128, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 83.715, "latency": 4252.235, "stderr": 1.601, "cost_per_test": 17.656424, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 82.688, "latency": 8791.194, "stderr": 1.997, "cost_per_test": 58.51711, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 82.273, "latency": 10480.572, "stderr": 1.698, "cost_per_test": 60.289268, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 81.909, "latency": 2771.488, "stderr": 1.502, "cost_per_test": 11.672651, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 78.676, "latency": 4389.65, "stderr": 1.786, "cost_per_test": 6.828942, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 77.644, "latency": 1764.488, "stderr": 1.689, "cost_per_test": 15.289359, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 76.803, "latency": 9447.46, "stderr": 1.914, "cost_per_test": 75.675797, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 76.304, "latency": 7602.078, "stderr": 2.42, "cost_per_test": 35.860741, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 75.701, "latency": 13078.924, "stderr": 1.812, "cost_per_test": 46.490946, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.6": { "accuracy": 74.029, "latency": 16044.462, "stderr": 1.851, "cost_per_test": 16.759019, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 72.345, "latency": 2016.05, "stderr": 1.92, "cost_per_test": 5.377882, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 72.067, "latency": 5471.639, "stderr": 1.789, "cost_per_test": 24.358317, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 71.999, "latency": 3709.707, "stderr": 1.987, "cost_per_test": 0.896935, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 71.945, "latency": 5523.099, "stderr": 1.828, "cost_per_test": 31.267424, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 71.899, "latency": 2799.268, "stderr": 1.905, "cost_per_test": 10.928509, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 71.722, "latency": 4112.719, "stderr": 1.823, "cost_per_test": 11.555338, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 71.714, "latency": 12091.441, "stderr": 1.952, "cost_per_test": 0.470526, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.5": { "accuracy": 70.775, "latency": 1322.959, "stderr": 1.822, "cost_per_test": 6.95454, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 70.103, "latency": 4970.276, "stderr": 1.87, "cost_per_test": 0.431698, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 69.815, "latency": 10825.24, "stderr": 1.87, "cost_per_test": 0.726582, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 69.169, "latency": 7801.049, "stderr": 2.032, "cost_per_test": 12.809118, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 68.709, "latency": 19703.41, "stderr": 2.193, "cost_per_test": 54.285113, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 68.663, "latency": 1290.211, "stderr": 2.011, "cost_per_test": 7.130366, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 68.596, "latency": 2386.411, "stderr": 1.792, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 68.291, "latency": 2493.897, "stderr": 1.873, "cost_per_test": 0.942725, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 66.252, "latency": 15354.08, "stderr": 2.237, "cost_per_test": 21.957364, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 66.223, "latency": 1734.863, "stderr": 1.807, "cost_per_test": 0.180944, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 65.246, "latency": 1139.043, "stderr": 1.81, "cost_per_test": 4.124197, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 64.225, "latency": 3246.698, "stderr": 1.894, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 62.768, "latency": 20494.929, "stderr": 2.458, "cost_per_test": 70.475273, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.2": { "accuracy": 62.605, "latency": 7588.932, "stderr": 2.026, "cost_per_test": 12.57696, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 61.922, "latency": 1598.124, "stderr": 1.793, "cost_per_test": 2.467822, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.4-2026-03-05-high": { "accuracy": 61.764, "latency": 958.495, "stderr": 1.791, "cost_per_test": 2.27429, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 59.926, "latency": 2852.749, "stderr": 1.947, "cost_per_test": 5.923527, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 59.103, "latency": 4349.841, "stderr": 1.672, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 57.291, "latency": 20327.596, "stderr": 2.386, "cost_per_test": 1.285061, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 56.356, "latency": 1104.517, "stderr": 1.942, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 54.375, "latency": 3225.344, "stderr": 1.954, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 50.879, "latency": 3931.735, "stderr": 1.697, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 49.435, "latency": 3860.332, "stderr": 1.799, "cost_per_test": 3.185154, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 48.609, "latency": 19801.258, "stderr": 2.482, "cost_per_test": 0.79862, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 47.952, "latency": 4794.524, "stderr": 1.955, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 47.778, "latency": 3778.098, "stderr": 1.636, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 42.528, "latency": 1710.815, "stderr": 1.632, "cost_per_test": 0.479317, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 39.971, "latency": 21263.689, "stderr": 2.7, "cost_per_test": 12.357107, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 39.468, "latency": 755.928, "stderr": 1.809, "cost_per_test": 2.629095, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 33.852, "latency": 434.999, "stderr": 1.758, "cost_per_test": 0.337485, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 33.83, "latency": 418.92, "stderr": 1.622, "cost_per_test": 0.439965, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 28.561, "latency": 1981.11, "stderr": 1.974, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 27.122, "latency": 20592.047, "stderr": 2.37, "cost_per_test": 13.063498, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 25.422, "latency": 457.113, "stderr": 1.636, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 24.381, "latency": 2734.146, "stderr": 1.86, "cost_per_test": 4.143489, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse-wonder-full": { "accuracy": 24.187, "latency": 21822.228, "stderr": 2.638, "cost_per_test": 28.645593, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.3": { "accuracy": 22.105, "latency": 236.134, "stderr": 1.522, "cost_per_test": 0.355823, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 18.153, "latency": 1465.949, "stderr": 1.09, "cost_per_test": 0.216901, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 17.15, "latency": 1541.675, "stderr": 1.333, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-m.1": { "accuracy": 15.833, "latency": 4921.898, "stderr": 1.474, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 14.282, "latency": 176.246, "stderr": 1.072, "cost_per_test": 0.063906, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 13.536, "latency": 1460.176, "stderr": 1.394, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 11.168, "latency": 21400.403, "stderr": 1.829, "cost_per_test": 17.990872, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 9.967, "latency": 3718.555, "stderr": 1.376, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null } }, "almost": { "anthropic/claude-opus-5-5": { "accuracy": 65.0, "tie_breaker_score": 86.986, "latency": 8660.211, "stderr": 3.381, "cost_per_test": 68.051144, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 53.5, "tie_breaker_score": 82.688, "latency": 8791.194, "stderr": 3.536, "cost_per_test": 58.51711, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 50.0, "tie_breaker_score": 85.417, "latency": 1621.514, "stderr": 3.544, "cost_per_test": 11.574605, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 49.5, "tie_breaker_score": 76.304, "latency": 7602.078, "stderr": 3.544, "cost_per_test": 35.860741, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 46.0, "tie_breaker_score": 85.25, "latency": 4223.187, "stderr": 3.533, "cost_per_test": 2.868128, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 41.5, "tie_breaker_score": 82.273, "latency": 10480.572, "stderr": 3.493, "cost_per_test": 60.289268, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 41.0, "tie_breaker_score": 83.715, "latency": 4252.235, "stderr": 3.487, "cost_per_test": 17.656424, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 33.0, "tie_breaker_score": 76.803, "latency": 9447.46, "stderr": 3.333, "cost_per_test": 75.675797, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 32.5, "tie_breaker_score": 81.909, "latency": 2771.488, "stderr": 3.32, "cost_per_test": 11.672651, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 29.5, "tie_breaker_score": 78.676, "latency": 4389.65, "stderr": 3.233, "cost_per_test": 6.828942, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 23.0, "tie_breaker_score": 77.644, "latency": 1764.488, "stderr": 2.983, "cost_per_test": 15.289359, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 21.5, "tie_breaker_score": 75.701, "latency": 13078.924, "stderr": 2.912, "cost_per_test": 46.490946, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.6": { "accuracy": 19.0, "tie_breaker_score": 74.029, "latency": 16044.462, "stderr": 2.781, "cost_per_test": 16.759019, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 19.0, "tie_breaker_score": 68.709, "latency": 19703.41, "stderr": 2.781, "cost_per_test": 54.285113, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 17.5, "tie_breaker_score": 71.714, "latency": 12091.441, "stderr": 2.694, "cost_per_test": 0.470526, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 17.5, "tie_breaker_score": 62.768, "latency": 20494.929, "stderr": 2.694, "cost_per_test": 70.475273, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 16.5, "tie_breaker_score": 71.999, "latency": 3709.707, "stderr": 2.631, "cost_per_test": 0.896935, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 16.5, "tie_breaker_score": 66.252, "latency": 15354.08, "stderr": 2.631, "cost_per_test": 21.957364, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 15.5, "tie_breaker_score": 71.945, "latency": 5523.099, "stderr": 2.565, "cost_per_test": 31.267424, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 15.0, "tie_breaker_score": 72.345, "latency": 2016.05, "stderr": 2.531, "cost_per_test": 5.377882, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 14.5, "tie_breaker_score": 69.169, "latency": 7801.049, "stderr": 2.496, "cost_per_test": 12.809118, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 14.0, "tie_breaker_score": 71.899, "latency": 2799.268, "stderr": 2.46, "cost_per_test": 10.928509, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 13.5, "tie_breaker_score": 72.067, "latency": 5471.639, "stderr": 2.422, "cost_per_test": 24.358317, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 13.5, "tie_breaker_score": 70.103, "latency": 4970.276, "stderr": 2.422, "cost_per_test": 0.431698, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 13.0, "tie_breaker_score": 71.722, "latency": 4112.719, "stderr": 2.384, "cost_per_test": 11.555338, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 12.5, "tie_breaker_score": 69.815, "latency": 10825.24, "stderr": 2.344, "cost_per_test": 0.726582, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 10.5, "tie_breaker_score": 39.971, "latency": 21263.689, "stderr": 2.173, "cost_per_test": 12.357107, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.5": { "accuracy": 9.5, "tie_breaker_score": 70.775, "latency": 1322.959, "stderr": 2.079, "cost_per_test": 6.95454, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 9.5, "tie_breaker_score": 68.663, "latency": 1290.211, "stderr": 2.079, "cost_per_test": 7.130366, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 9.5, "tie_breaker_score": 64.225, "latency": 3246.698, "stderr": 2.079, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 9.5, "tie_breaker_score": 62.605, "latency": 7588.932, "stderr": 2.079, "cost_per_test": 12.57696, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 9.0, "tie_breaker_score": 68.291, "latency": 2493.897, "stderr": 2.029, "cost_per_test": 0.942725, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 8.5, "tie_breaker_score": 57.291, "latency": 20327.596, "stderr": 1.977, "cost_per_test": 1.285061, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 8.0, "tie_breaker_score": 66.223, "latency": 1734.863, "stderr": 1.923, "cost_per_test": 0.180944, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 8.0, "tie_breaker_score": 24.187, "latency": 21822.228, "stderr": 1.923, "cost_per_test": 28.645593, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 7.0, "tie_breaker_score": 68.596, "latency": 2386.411, "stderr": 1.809, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 7.0, "tie_breaker_score": 48.609, "latency": 19801.258, "stderr": 1.809, "cost_per_test": 0.79862, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.5": { "accuracy": 4.5, "tie_breaker_score": 65.246, "latency": 1139.043, "stderr": 1.47, "cost_per_test": 4.124197, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-2026-03-05-high": { "accuracy": 3.5, "tie_breaker_score": 61.764, "latency": 958.495, "stderr": 1.303, "cost_per_test": 2.27429, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 3.0, "tie_breaker_score": 54.375, "latency": 3225.344, "stderr": 1.209, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 2.5, "tie_breaker_score": 61.922, "latency": 1598.124, "stderr": 1.107, "cost_per_test": 2.467822, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 2.5, "tie_breaker_score": 59.926, "latency": 2852.749, "stderr": 1.107, "cost_per_test": 5.923527, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 2.5, "tie_breaker_score": 27.122, "latency": 20592.047, "stderr": 1.107, "cost_per_test": 13.063498, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 2.0, "tie_breaker_score": 59.103, "latency": 4349.841, "stderr": 0.992, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 1.5, "tie_breaker_score": 56.356, "latency": 1104.517, "stderr": 0.862, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 1.0, "tie_breaker_score": 50.879, "latency": 3931.735, "stderr": 0.705, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 1.0, "tie_breaker_score": 39.468, "latency": 755.928, "stderr": 0.705, "cost_per_test": 2.629095, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 1.0, "tie_breaker_score": 28.561, "latency": 1981.11, "stderr": 0.705, "cost_per_test": null, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 1.0, "tie_breaker_score": 11.168, "latency": 21400.403, "stderr": 0.705, "cost_per_test": 17.990872, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.7-code": { "accuracy": 0.5, "tie_breaker_score": 49.435, "latency": 3860.332, "stderr": 0.5, "cost_per_test": 3.185154, "temperature": null, "top_p": 0.95, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.5, "tie_breaker_score": 47.952, "latency": 4794.524, "stderr": 0.5, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.5, "tie_breaker_score": 42.528, "latency": 1710.815, "stderr": 0.5, "cost_per_test": 0.479317, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.5, "tie_breaker_score": 9.967, "latency": 3718.555, "stderr": 0.5, "cost_per_test": null, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "tie_breaker_score": 47.778, "latency": 3778.098, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "tie_breaker_score": 33.852, "latency": 434.999, "stderr": 0.0, "cost_per_test": 0.337485, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "tie_breaker_score": 33.83, "latency": 418.92, "stderr": 0.0, "cost_per_test": 0.439965, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "tie_breaker_score": 25.422, "latency": 457.113, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "tie_breaker_score": 24.381, "latency": 2734.146, "stderr": 0.0, "cost_per_test": 4.143489, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "tie_breaker_score": 22.105, "latency": 236.134, "stderr": 0.0, "cost_per_test": 0.355823, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "tie_breaker_score": 18.153, "latency": 1465.949, "stderr": 0.0, "cost_per_test": 0.216901, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "tie_breaker_score": 17.15, "latency": 1541.675, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "tie_breaker_score": 15.833, "latency": 4921.898, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "tie_breaker_score": 14.282, "latency": 176.246, "stderr": 0.0, "cost_per_test": 0.063906, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "tie_breaker_score": 13.536, "latency": 1460.176, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null } } } }