{ "metadata": { "benchmark": "Legal Research Bench", "slug": "legal_research", "description": "Evaluating agents on legal research tasks across diverse areas of US law", "benchmark_id": "legal_research", "family": "legal_research", "version": "1", "updated": "2026-11-05", "dataset_type": "private", "industry": "legal", "tasks": { "overall": "Overall", "administrative_regulatory": "Administrative / Regulatory", "business_commercial": "Business & Commercial", "civil_litigation": "Civil Litigation", "constitutional_civil_rights": "Constitutional / Civil Rights", "criminal": "Criminal", "family": "Family", "health": "Health", "immigration": "Immigration", "weighted_pass": "Weighted Pass", "weighted_pass__administrative_regulatory": "Administrative / Regulatory", "weighted_pass__business_commercial": "Business & Commercial", "weighted_pass__civil_litigation": "Civil Litigation", "weighted_pass__constitutional_civil_rights": "Constitutional / Civil Rights", "weighted_pass__criminal": "Criminal", "weighted_pass__family": "Family", "weighted_pass__health": "Health", "weighted_pass__immigration": "Immigration" }, "models": [ "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.7-plus", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "alibaba/qwen3.8-max-internal", "ant/ling-3.0-flash-2607", "ant/ling-3.0-flash-af-rc3", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "baseten/nemotron-3.5-super-vl-120b-a12b", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "deepseek/deepseek-v4.1-flash-1017", "fireworks/ember-1", "fireworks/nemotron-lightning-3p5-30b-a3b", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "google/gemini-4-carbon", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-4.7", "inception/mercury-2.5", "inception/mercury-3", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k3", "kimi/kimi-k3.1-pro", "meta/muse-spark-open", "meta/muse-wonder", "meta/muse-wonder-full", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "minimax/MiniMax-M3.1-flash", "mistralai/mistral-large-4", "mistralai/mistral-large-4-oss", "mistralai/mistral-medium-3.5", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "openai/gpt-6.1-terra", "poolside/laguna-m.1", "poolside/laguna-xs.2", "reflection/Beam-1.0", "stepfun/step-5-preview", "tencent/hy4", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-2-small", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.1", "zai/glm-5.2", "zai/glm-5.3", "zai/glm-5.3-flash", "zai/glm-legal" ], "partners": [ { "name": "fisher_phillips", "url": "https://www.fisherphillips.com/" }, { "name": "mcdermott", "url": "https://www.mcdermottlaw.com/" }, { "name": "reed_smith", "url": "https://www.reedsmith.com/" }, { "name": "legaltech_hub", "url": "https://www.legaltechnologyhub.com/" } ], "showBadge": false, "visible": true, "use_cost_per_test": true, "token_tasks_measured": 208, "runner": "external", "mode": "agentic", "archived": false, "partner": false, "total_models": 91 }, "tasks": { "overall": { "google/gemini-4-carbon": { "accuracy": 55.769, "latency": 3112.579, "stderr": 3.452, "cost_per_test": 8.743524, "token_totals": { "input_tokens": 1062929092, "output_tokens": 52772544, "reasoning_tokens": 36254828, "cache_read_tokens": 918030058, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 39.2, "avg_query_seconds": 79.5, "median_query_seconds": 33.5, "avg_ttft_seconds": 12.7, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 55.288, "latency": 317.461, "stderr": 3.456, "cost_per_test": 0.588812, "token_totals": { "input_tokens": 240995118, "output_tokens": 10646285, "reasoning_tokens": 6279342, "cache_read_tokens": 205258324, "cache_write_tokens": 0 }, "token_tasks_measured": 205, "turn_metrics": { "tasks_measured": 208, "avg_turns": 25.0, "avg_query_seconds": 12.5, "median_query_seconds": 4.6, "avg_ttft_seconds": 4.9, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 55.288, "latency": 1860.719, "stderr": 3.456, "cost_per_test": 6.762498, "token_totals": { "input_tokens": 626960894, "output_tokens": 23871466, "reasoning_tokens": 9873124, "cache_read_tokens": 540621000, "cache_write_tokens": 86242293 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 37.0, "avg_query_seconds": 50.3, "median_query_seconds": 27.5, "avg_ttft_seconds": 14.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 55.288, "latency": 3449.382, "stderr": 3.456, "cost_per_test": 23.055789, "token_totals": { "input_tokens": 1085827683, "output_tokens": 50734245, "reasoning_tokens": 31513293, "cache_read_tokens": 927334456, "cache_write_tokens": 158405763 }, "token_tasks_measured": 206, "turn_metrics": { "tasks_measured": 206, "avg_turns": 48.5, "avg_query_seconds": 71.1, "median_query_seconds": 16.8, "avg_ttft_seconds": 14.4, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 54.808, "latency": 1881.045, "stderr": 3.459, "cost_per_test": 6.44908, "token_totals": { "input_tokens": 915877177, "output_tokens": 24432059, "reasoning_tokens": 11886477, "cache_read_tokens": 739668768, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 38.2, "avg_query_seconds": 49.2, "median_query_seconds": 16.4, "avg_ttft_seconds": 8.1, "output_time_split": { "reasoning_pct": 79.0, "content_pct": 3.0, "tool_call_pct": 18.1 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 53.365, "latency": 1654.378, "stderr": 3.467, "cost_per_test": 3.595401, "token_totals": { "input_tokens": 760777228, "output_tokens": 19689738, "reasoning_tokens": 13012367, "cache_read_tokens": 707647460, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 45.3, "avg_query_seconds": 36.5, "median_query_seconds": 9.9, "avg_ttft_seconds": 8.2, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 50.481, "latency": 927.111, "stderr": 3.475, "cost_per_test": 3.980242, "token_totals": { "input_tokens": 732491974, "output_tokens": 17268053, "reasoning_tokens": 11668590, "cache_read_tokens": 644062808, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 46.7, "avg_query_seconds": 19.9, "median_query_seconds": 7.2, "avg_ttft_seconds": 5.5, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 50.481, "latency": 6749.531, "stderr": 3.475, "cost_per_test": 23.613691, "token_totals": { "input_tokens": 1686854010, "output_tokens": 141554772, "reasoning_tokens": 123959393, "cache_read_tokens": 1436953409, "cache_write_tokens": 249809865 }, "token_tasks_measured": 185, "turn_metrics": { "tasks_measured": 197, "avg_turns": 72.9, "avg_query_seconds": 89.0, "median_query_seconds": 9.2, "avg_ttft_seconds": 9.7, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 49.519, "latency": 1337.604, "stderr": 3.475, "cost_per_test": 9.791426, "token_totals": { "input_tokens": 210052964, "output_tokens": 22537892, "reasoning_tokens": null, "cache_read_tokens": 150055095, "cache_write_tokens": 59958741 }, "token_tasks_measured": 207, "turn_metrics": { "tasks_measured": 207, "avg_turns": 14.6, "avg_query_seconds": 91.5, "median_query_seconds": 56.5, "avg_ttft_seconds": 37.0, "output_time_split": { "reasoning_pct": 0.0, "content_pct": 0.8, "tool_call_pct": 99.2 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 49.038, "latency": 3110.171, "stderr": 3.475, "cost_per_test": 2.241388, "token_totals": { "input_tokens": 458279584, "output_tokens": 55494507, "reasoning_tokens": null, "cache_read_tokens": 368033787, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 23.3, "avg_query_seconds": 133.6, "median_query_seconds": 67.1, "avg_ttft_seconds": 3.4, "output_time_split": { "reasoning_pct": 82.2, "content_pct": 1.1, "tool_call_pct": 16.7 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.6": { "accuracy": 48.077, "latency": 1504.077, "stderr": 3.473, "cost_per_test": 1.530204, "token_totals": { "input_tokens": 274776010, "output_tokens": 12377024, "reasoning_tokens": 7368696, "cache_read_tokens": 203687808, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 20.9, "avg_query_seconds": 72.1, "median_query_seconds": 34.7, "avg_ttft_seconds": 17.7, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 48.077, "latency": 4631.383, "stderr": 3.473, "cost_per_test": 21.605566, "token_totals": { "input_tokens": 1487117357, "output_tokens": 70127005, "reasoning_tokens": 61207392, "cache_read_tokens": 1241425920, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 47.3, "avg_query_seconds": 97.8, "median_query_seconds": 10.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 48.077, "latency": 5588.989, "stderr": 3.473, "cost_per_test": 13.663179, "token_totals": { "input_tokens": 2650038387, "output_tokens": 173348390, "reasoning_tokens": 151205430, "cache_read_tokens": 2338418360, "cache_write_tokens": 311466299 }, "token_tasks_measured": 201, "turn_metrics": { "tasks_measured": 204, "avg_turns": 85.2, "avg_query_seconds": 64.6, "median_query_seconds": 11.2, "avg_ttft_seconds": 8.2, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 47.596, "latency": 3677.636, "stderr": 3.471, "cost_per_test": 2.486351, "token_totals": { "input_tokens": 544286180, "output_tokens": 42522887, "reasoning_tokens": 35664649, "cache_read_tokens": 472313472, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 35.5, "avg_query_seconds": 103.5, "median_query_seconds": 58.3, "avg_ttft_seconds": 27.2, "output_time_split": { "reasoning_pct": 76.7, "content_pct": 1.1, "tool_call_pct": 22.1 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.7": { "accuracy": 47.115, "latency": 1274.337, "stderr": 3.469, "cost_per_test": 4.920812, "token_totals": { "input_tokens": 476673717, "output_tokens": 14028919, "reasoning_tokens": 9121242, "cache_read_tokens": 350504320, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 44.0, "avg_query_seconds": 28.9, "median_query_seconds": 12.5, "avg_ttft_seconds": 3.3, "output_time_split": { "reasoning_pct": 78.7, "content_pct": 12.9, "tool_call_pct": 8.5 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 47.115, "latency": 1816.322, "stderr": 3.469, "cost_per_test": 0.181201, "token_totals": { "input_tokens": 246448526, "output_tokens": 20667764, "reasoning_tokens": 15354087, "cache_read_tokens": 204079360, "cache_write_tokens": 0 }, "token_tasks_measured": 205, "turn_metrics": { "tasks_measured": 206, "avg_turns": 21.3, "avg_query_seconds": 85.4, "median_query_seconds": 43.3, "avg_ttft_seconds": 5.2, "output_time_split": { "reasoning_pct": 81.4, "content_pct": 1.9, "tool_call_pct": 16.6 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 46.635, "latency": 735.191, "stderr": 3.467, "cost_per_test": 0.288586, "token_totals": { "input_tokens": 701422864, "output_tokens": 24259680, "reasoning_tokens": null, "cache_read_tokens": 646891008, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 48.5, "avg_query_seconds": 15.1, "median_query_seconds": 4.4, "avg_ttft_seconds": 1.8, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k3": { "accuracy": 46.154, "latency": 760.122, "stderr": 3.465, "cost_per_test": 3.469601, "token_totals": { "input_tokens": 672994446, "output_tokens": 6248749, "reasoning_tokens": 3873135, "cache_read_tokens": 651592955, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 37.2, "avg_query_seconds": 20.4, "median_query_seconds": 10.2, "avg_ttft_seconds": 4.2, "output_time_split": { "reasoning_pct": 61.9, "content_pct": 2.6, "tool_call_pct": 35.5 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 46.154, "latency": 7649.584, "stderr": 3.465, "cost_per_test": 5.312295, "token_totals": { "input_tokens": 1691431541, "output_tokens": 63439847, "reasoning_tokens": 53571199, "cache_read_tokens": 1582915968, "cache_write_tokens": null }, "token_tasks_measured": 187, "turn_metrics": { "tasks_measured": 187, "avg_turns": 72.2, "avg_query_seconds": 105.9, "median_query_seconds": 44.3, "avg_ttft_seconds": 21.5, "output_time_split": { "reasoning_pct": 81.4, "content_pct": 0.2, "tool_call_pct": 18.5 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 45.192, "latency": 2106.881, "stderr": 3.459, "cost_per_test": 0.69856, "token_totals": { "input_tokens": 2003160078, "output_tokens": 87441264, "reasoning_tokens": 76652436, "cache_read_tokens": 1909100928, "cache_write_tokens": 0 }, "token_tasks_measured": 207, "turn_metrics": { "tasks_measured": 207, "avg_turns": 74.5, "avg_query_seconds": 28.2, "median_query_seconds": 6.4, "avg_ttft_seconds": 1.6, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 45.192, "latency": 3067.486, "stderr": 3.459, "cost_per_test": 0.240392, "token_totals": { "input_tokens": 636228822, "output_tokens": 43453265, "reasoning_tokens": null, "cache_read_tokens": 559661440, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 30.9, "avg_query_seconds": 99.2, "median_query_seconds": 40.1, "avg_ttft_seconds": 1.8, "output_time_split": { "reasoning_pct": 85.5, "content_pct": 0.9, "tool_call_pct": 13.6 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 45.192, "latency": 3839.511, "stderr": 3.459, "cost_per_test": 0.846372, "token_totals": { "input_tokens": 312749338, "output_tokens": 50001075, "reasoning_tokens": 43836379, "cache_read_tokens": 264949824, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 31.0, "avg_query_seconds": 123.9, "median_query_seconds": 36.1, "avg_ttft_seconds": 3.4, "output_time_split": { "reasoning_pct": 72.0, "content_pct": 8.7, "tool_call_pct": 19.3 } }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "fireworks/ember-1": { "accuracy": 44.712, "latency": 2313.552, "stderr": 3.456, "cost_per_test": 3.674091, "token_totals": { "input_tokens": 899320434, "output_tokens": 18199298, "reasoning_tokens": 11913573, "cache_read_tokens": 817311062, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 40.2, "avg_query_seconds": 57.5, "median_query_seconds": 21.2, "avg_ttft_seconds": 6.0, "output_time_split": { "reasoning_pct": 50.2, "content_pct": 2.3, "tool_call_pct": 47.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 43.75, "latency": 306.893, "stderr": 3.448, "cost_per_test": 0.508097, "token_totals": { "input_tokens": 163340498, "output_tokens": 5837394, "reasoning_tokens": 2676881, "cache_read_tokens": 113477050, "cache_write_tokens": 0 }, "token_tasks_measured": 205, "turn_metrics": { "tasks_measured": 205, "avg_turns": 18.1, "avg_query_seconds": 17.0, "median_query_seconds": 6.2, "avg_ttft_seconds": 6.3, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 43.75, "latency": 714.036, "stderr": 3.448, "cost_per_test": 2.820942, "token_totals": { "input_tokens": 136489421, "output_tokens": 12617848, "reasoning_tokens": null, "cache_read_tokens": 98212592, "cache_write_tokens": 38178928 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 11.7, "avg_query_seconds": 61.7, "median_query_seconds": 48.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 43.75, "latency": 4979.806, "stderr": 3.448, "cost_per_test": 1.647359, "token_totals": { "input_tokens": 1674824268, "output_tokens": 253184033, "reasoning_tokens": 237163492, "cache_read_tokens": 1419547906, "cache_write_tokens": 255154225 }, "token_tasks_measured": 206, "turn_metrics": { "tasks_measured": 206, "avg_turns": 60.8, "avg_query_seconds": 81.9, "median_query_seconds": 14.1, "avg_ttft_seconds": 8.8, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 41.827, "latency": 1545.997, "stderr": 3.428, "cost_per_test": 2.719446, "token_totals": { "input_tokens": 435277342, "output_tokens": 29025915, "reasoning_tokens": null, "cache_read_tokens": 353355756, "cache_write_tokens": 81742641 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 24.7, "avg_query_seconds": 62.6, "median_query_seconds": 26.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-legal": { "accuracy": 41.827, "latency": 2388.741, "stderr": 3.428, "cost_per_test": 0.04878, "token_totals": { "input_tokens": 203583535, "output_tokens": 5833302, "reasoning_tokens": 4185422, "cache_read_tokens": 177157760, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 22.1, "avg_query_seconds": 108.1, "median_query_seconds": 61.1, "avg_ttft_seconds": 9.4, "output_time_split": { "reasoning_pct": 65.0, "content_pct": 0.8, "tool_call_pct": 34.2 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 41.346, "latency": 701.783, "stderr": 3.423, "cost_per_test": 0.24793, "token_totals": { "input_tokens": 410443550, "output_tokens": 29229774, "reasoning_tokens": 25253265, "cache_read_tokens": 362718592, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 26.1, "avg_query_seconds": 26.9, "median_query_seconds": 11.6, "avg_ttft_seconds": 2.1, "output_time_split": { "reasoning_pct": 76.9, "content_pct": 1.1, "tool_call_pct": 22.0 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 41.346, "latency": 4172.869, "stderr": 3.423, "cost_per_test": 7.907299, "token_totals": { "input_tokens": 1570540015, "output_tokens": 81682466, "reasoning_tokens": 73169720, "cache_read_tokens": 1375861858, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 46.6, "avg_query_seconds": 89.6, "median_query_seconds": 11.8, "avg_ttft_seconds": 5.2, "output_time_split": { "reasoning_pct": 48.7, "content_pct": 3.0, "tool_call_pct": 48.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4": { "accuracy": 41.346, "latency": 5455.029, "stderr": 3.423, "cost_per_test": 2.222804, "token_totals": { "input_tokens": 1641853555, "output_tokens": 106544013, "reasoning_tokens": 94881685, "cache_read_tokens": 1520894592, "cache_write_tokens": 0 }, "token_tasks_measured": 194, "turn_metrics": { "tasks_measured": 202, "avg_turns": 68.1, "avg_query_seconds": 79.1, "median_query_seconds": 9.9, "avg_ttft_seconds": 3.8, "output_time_split": null }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 40.865, "latency": 530.813, "stderr": 3.417, "cost_per_test": 0.650063, "token_totals": { "input_tokens": 193686912, "output_tokens": 8979986, "reasoning_tokens": 5305335, "cache_read_tokens": 133646110, "cache_write_tokens": 0 }, "token_tasks_measured": 205, "turn_metrics": { "tasks_measured": 208, "avg_turns": 20.6, "avg_query_seconds": 25.4, "median_query_seconds": 9.5, "avg_ttft_seconds": 4.5, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 40.865, "latency": 2527.177, "stderr": 3.417, "cost_per_test": 1.088017, "token_totals": { "input_tokens": 433434003, "output_tokens": 35408474, "reasoning_tokens": 30213976, "cache_read_tokens": 380911360, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 30.2, "avg_query_seconds": 83.7, "median_query_seconds": 27.7, "avg_ttft_seconds": 2.4, "output_time_split": { "reasoning_pct": 76.2, "content_pct": 1.7, "tool_call_pct": 22.1 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "stepfun/step-5-preview": { "accuracy": 40.385, "latency": 2168.413, "stderr": 3.41, "cost_per_test": 0.838645, "token_totals": { "input_tokens": 683339325, "output_tokens": 23833353, "reasoning_tokens": 0, "cache_read_tokens": 604305152, "cache_write_tokens": 0 }, "token_tasks_measured": 207, "turn_metrics": { "tasks_measured": 207, "avg_turns": 47.8, "avg_query_seconds": 45.3, "median_query_seconds": 17.0, "avg_ttft_seconds": 4.2, "output_time_split": { "reasoning_pct": 69.1, "content_pct": 2.5, "tool_call_pct": 28.4 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-5.5": { "accuracy": 40.385, "latency": 2265.105, "stderr": 3.41, "cost_per_test": 7.402471, "token_totals": { "input_tokens": 743971258, "output_tokens": 21289365, "reasoning_tokens": 16694105, "cache_read_tokens": 647774464, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 60.3, "avg_query_seconds": 38.0, "median_query_seconds": 6.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 39.423, "latency": 1491.339, "stderr": 3.397, "cost_per_test": 10.577143, "token_totals": { "input_tokens": 290661092, "output_tokens": 21448775, "reasoning_tokens": 16238030, "cache_read_tokens": 215071044, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 19.0, "avg_query_seconds": 78.6, "median_query_seconds": 31.7, "avg_ttft_seconds": 9.4, "output_time_split": { "reasoning_pct": 25.5, "content_pct": 0.6, "tool_call_pct": 74.0 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 38.942, "latency": 324.318, "stderr": 3.389, "cost_per_test": 1.630498, "token_totals": { "input_tokens": 581263116, "output_tokens": 14420656, "reasoning_tokens": 10782340, "cache_read_tokens": 474745174, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 50.6, "avg_query_seconds": 6.4, "median_query_seconds": 1.8, "avg_ttft_seconds": 1.3, "output_time_split": { "reasoning_pct": 98.4, "content_pct": 0.0, "tool_call_pct": 1.6 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 38.462, "latency": 511.955, "stderr": 3.381, "cost_per_test": 3.05609, "token_totals": { "input_tokens": 280128756, "output_tokens": 7275719, "reasoning_tokens": 2846795, "cache_read_tokens": 225999114, "cache_write_tokens": 53655955 }, "token_tasks_measured": 207, "turn_metrics": { "tasks_measured": 208, "avg_turns": 19.8, "avg_query_seconds": 25.9, "median_query_seconds": 13.9, "avg_ttft_seconds": 8.5, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 38.462, "latency": 989.604, "stderr": 3.381, "cost_per_test": 2.302948, "token_totals": { "input_tokens": 294037202, "output_tokens": 10695335, "reasoning_tokens": null, "cache_read_tokens": 223683365, "cache_write_tokens": 70074508 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 24.6, "avg_query_seconds": 40.0, "median_query_seconds": 20.9, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 38.462, "latency": 1763.023, "stderr": 3.381, "cost_per_test": 1.559385, "token_totals": { "input_tokens": 1351006836, "output_tokens": 6986771, "reasoning_tokens": 3248800, "cache_read_tokens": 1267363808, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 31.6, "avg_query_seconds": 55.8, "median_query_seconds": 10.0, "avg_ttft_seconds": 11.0, "output_time_split": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 38.462, "latency": 2060.48, "stderr": 3.381, "cost_per_test": 3.384561, "token_totals": { "input_tokens": 329856713, "output_tokens": 20763888, "reasoning_tokens": 15434407, "cache_read_tokens": 260063643, "cache_write_tokens": 68892844 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 21.7, "avg_query_seconds": 95.1, "median_query_seconds": 43.9, "avg_ttft_seconds": 13.1, "output_time_split": { "reasoning_pct": 24.3, "content_pct": 0.9, "tool_call_pct": 74.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 37.981, "latency": 312.994, "stderr": 3.373, "cost_per_test": 0.377379, "token_totals": { "input_tokens": 126240889, "output_tokens": 5901711, "reasoning_tokens": 3106623, "cache_read_tokens": 94898720, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 16.0, "avg_query_seconds": 19.6, "median_query_seconds": 9.4, "avg_ttft_seconds": 4.1, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 37.981, "latency": 919.689, "stderr": 3.373, "cost_per_test": 0.070424, "token_totals": { "input_tokens": 365192541, "output_tokens": 17884630, "reasoning_tokens": 10104423, "cache_read_tokens": 305459072, "cache_write_tokens": 0 }, "token_tasks_measured": 202, "turn_metrics": { "tasks_measured": 205, "avg_turns": 27.4, "avg_query_seconds": 35.9, "median_query_seconds": 19.2, "avg_ttft_seconds": 3.5, "output_time_split": { "reasoning_pct": 0.7, "content_pct": 0.0, "tool_call_pct": 99.3 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.5": { "accuracy": 37.981, "latency": 953.767, "stderr": 3.373, "cost_per_test": 1.086814, "token_totals": { "input_tokens": 128982108, "output_tokens": 6582496, "reasoning_tokens": 3272884, "cache_read_tokens": 74664576, "cache_write_tokens": 0 }, "token_tasks_measured": 207, "turn_metrics": { "tasks_measured": 207, "avg_turns": 12.5, "avg_query_seconds": 76.2, "median_query_seconds": 26.8, "avg_ttft_seconds": 16.0, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 37.5, "latency": 6237.394, "stderr": 3.365, "cost_per_test": 7.231463, "token_totals": { "input_tokens": 1110222674, "output_tokens": 53232219, "reasoning_tokens": 44000137, "cache_read_tokens": 842326272, "cache_write_tokens": 266448109 }, "token_tasks_measured": 171, "turn_metrics": { "tasks_measured": 171, "avg_turns": 44.1, "avg_query_seconds": 141.6, "median_query_seconds": 25.7, "avg_ttft_seconds": 7.8, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 36.538, "latency": 2373.859, "stderr": 3.347, "cost_per_test": 0.845008, "token_totals": { "input_tokens": 1755111665, "output_tokens": 88330527, "reasoning_tokens": 78699683, "cache_read_tokens": 1562540626, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 53.5, "avg_query_seconds": 44.3, "median_query_seconds": 7.5, "avg_ttft_seconds": 3.4, "output_time_split": { "reasoning_pct": 45.7, "content_pct": 1.1, "tool_call_pct": 53.2 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 36.058, "latency": 1600.721, "stderr": 3.337, "cost_per_test": 2.159719, "token_totals": { "input_tokens": 600057696, "output_tokens": 49730866, "reasoning_tokens": 42100615, "cache_read_tokens": 0, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 34.9, "avg_query_seconds": 45.8, "median_query_seconds": 17.0, "avg_ttft_seconds": 3.1, "output_time_split": { "reasoning_pct": 78.0, "content_pct": 1.5, "tool_call_pct": 20.5 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 35.577, "latency": 1704.14, "stderr": 3.328, "cost_per_test": 1.09052, "token_totals": { "input_tokens": 803000611, "output_tokens": 18904714, "reasoning_tokens": null, "cache_read_tokens": 595193856, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 30.3, "avg_query_seconds": 56.2, "median_query_seconds": 24.4, "avg_ttft_seconds": 4.6, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 34.615, "latency": 288.96, "stderr": 3.307, "cost_per_test": 0.760397, "token_totals": { "input_tokens": 216485075, "output_tokens": 6173336, "reasoning_tokens": 3838872, "cache_read_tokens": 157677752, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 29.7, "avg_query_seconds": 9.7, "median_query_seconds": 2.8, "avg_ttft_seconds": 2.3, "output_time_split": { "reasoning_pct": 98.0, "content_pct": 0.0, "tool_call_pct": 2.0 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 34.135, "latency": 2911.983, "stderr": 3.296, "cost_per_test": 0.444777, "token_totals": { "input_tokens": 1230685744, "output_tokens": 102073269, "reasoning_tokens": 90583933, "cache_read_tokens": 1135801046, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 54.4, "avg_query_seconds": 53.5, "median_query_seconds": 16.7, "avg_ttft_seconds": 1.3, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 31.731, "latency": 5571.099, "stderr": 3.235, "cost_per_test": 2.723732, "token_totals": { "input_tokens": 1494014814, "output_tokens": 37508841, "reasoning_tokens": null, "cache_read_tokens": 885782016, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 46.9, "avg_query_seconds": 118.8, "median_query_seconds": 45.4, "avg_ttft_seconds": 10.6, "output_time_split": { "reasoning_pct": 88.6, "content_pct": 11.2, "tool_call_pct": 0.2 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-5.2": { "accuracy": 31.25, "latency": 1023.468, "stderr": 3.222, "cost_per_test": 0.889209, "token_totals": { "input_tokens": 268943504, "output_tokens": 12537934, "reasoning_tokens": 9395692, "cache_read_tokens": 210799936, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 23.3, "avg_query_seconds": 44.9, "median_query_seconds": 24.0, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 30.769, "latency": 277.753, "stderr": 3.208, "cost_per_test": 1.305281, "token_totals": { "input_tokens": 556323486, "output_tokens": 7365405, "reasoning_tokens": 5568107, "cache_read_tokens": 472254997, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 45.5, "avg_query_seconds": 5.9, "median_query_seconds": 2.6, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 30.288, "latency": 1266.744, "stderr": 3.194, "cost_per_test": 0.275172, "token_totals": { "input_tokens": 310561183, "output_tokens": 24287114, "reasoning_tokens": 20969113, "cache_read_tokens": 261666816, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 26.3, "avg_query_seconds": 48.2, "median_query_seconds": 18.4, "avg_ttft_seconds": 3.9, "output_time_split": { "reasoning_pct": 75.4, "content_pct": 2.6, "tool_call_pct": 21.9 } }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 30.288, "latency": 2547.747, "stderr": 3.194, "cost_per_test": 0.440296, "token_totals": { "input_tokens": 1143721770, "output_tokens": 59694127, "reasoning_tokens": 54603753, "cache_read_tokens": 1042899991, "cache_write_tokens": 99842719 }, "token_tasks_measured": 206, "turn_metrics": { "tasks_measured": 208, "avg_turns": 38.0, "avg_query_seconds": 67.1, "median_query_seconds": 16.9, "avg_ttft_seconds": 7.2, "output_time_split": { "reasoning_pct": 66.4, "content_pct": 0.0, "tool_call_pct": 33.6 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 29.808, "latency": 813.838, "stderr": 3.179, "cost_per_test": 0.342955, "token_totals": { "input_tokens": 273575187, "output_tokens": 6440429, "reasoning_tokens": null, "cache_read_tokens": 219245518, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 26.9, "avg_query_seconds": 30.5, "median_query_seconds": 12.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "reflection/Beam-1.0": { "accuracy": 28.846, "latency": 94.568, "stderr": 3.149, "cost_per_test": null, "token_totals": { "input_tokens": 394030584, "output_tokens": 5402310, "reasoning_tokens": 3403372, "cache_read_tokens": 359513792, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 36.4, "avg_query_seconds": 2.6, "median_query_seconds": 1.3, "avg_ttft_seconds": 0.4, "output_time_split": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "inception/mercury-3": { "accuracy": 28.846, "latency": 446.517, "stderr": 3.149, "cost_per_test": 0.074351, "token_totals": { "input_tokens": 271885585, "output_tokens": 22851545, "reasoning_tokens": 20198714, "cache_read_tokens": 219955154, "cache_write_tokens": 0 }, "token_tasks_measured": 207, "turn_metrics": { "tasks_measured": 208, "avg_turns": 26.3, "avg_query_seconds": 16.9, "median_query_seconds": 6.1, "avg_ttft_seconds": 3.3, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-6-sol": { "accuracy": 28.846, "latency": 1449.451, "stderr": 3.149, "cost_per_test": 4.897717, "token_totals": { "input_tokens": 709313788, "output_tokens": 26332564, "reasoning_tokens": 22035035, "cache_read_tokens": 634566483, "cache_write_tokens": 73881332 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 35.7, "avg_query_seconds": 40.6, "median_query_seconds": 10.6, "avg_ttft_seconds": 5.5, "output_time_split": { "reasoning_pct": 33.1, "content_pct": 2.4, "tool_call_pct": 64.5 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 28.365, "latency": 869.377, "stderr": 3.133, "cost_per_test": 0.494044, "token_totals": { "input_tokens": 271552348, "output_tokens": 8856526, "reasoning_tokens": null, "cache_read_tokens": 246578560, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 37.8, "avg_query_seconds": 23.0, "median_query_seconds": 4.5, "avg_ttft_seconds": 2.7, "output_time_split": { "reasoning_pct": 99.9, "content_pct": 0.0, "tool_call_pct": 0.1 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5.1": { "accuracy": 27.885, "latency": 325.824, "stderr": 3.117, "cost_per_test": 0.385737, "token_totals": { "input_tokens": 185419655, "output_tokens": 2414644, "reasoning_tokens": 0, "cache_read_tokens": 140363008, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 25.1, "avg_query_seconds": 12.9, "median_query_seconds": 6.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 25.962, "latency": 4202.728, "stderr": 3.047, "cost_per_test": 3.220614, "token_totals": { "input_tokens": 1392458117, "output_tokens": 69708508, "reasoning_tokens": 62148334, "cache_read_tokens": 1260829967, "cache_write_tokens": 130975483 }, "token_tasks_measured": 202, "turn_metrics": { "tasks_measured": 202, "avg_turns": 35.9, "avg_query_seconds": 117.1, "median_query_seconds": 21.4, "avg_ttft_seconds": 8.0, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 25.481, "latency": 774.823, "stderr": 3.029, "cost_per_test": 0.141548, "token_totals": { "input_tokens": 209190661, "output_tokens": 10262319, "reasoning_tokens": null, "cache_read_tokens": 190714496, "cache_write_tokens": 0 }, "token_tasks_measured": 207, "turn_metrics": { "tasks_measured": 207, "avg_turns": 34.6, "avg_query_seconds": 22.4, "median_query_seconds": 5.1, "avg_ttft_seconds": 1.9, "output_time_split": { "reasoning_pct": 100.0, "content_pct": 0.0, "tool_call_pct": 0.0 } }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 25.481, "latency": 808.802, "stderr": 3.029, "cost_per_test": 1.400701, "token_totals": { "input_tokens": 158509885, "output_tokens": 6696005, "reasoning_tokens": 4196744, "cache_read_tokens": 68025984, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 19.7, "avg_query_seconds": 41.6, "median_query_seconds": 10.1, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 25.0, "latency": 168.482, "stderr": 3.01, "cost_per_test": 0.466741, "token_totals": { "input_tokens": 116699947, "output_tokens": 4157671, "reasoning_tokens": 2875086, "cache_read_tokens": 80852090, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 20.1, "avg_query_seconds": 8.4, "median_query_seconds": 3.2, "avg_ttft_seconds": 1.5, "output_time_split": { "reasoning_pct": 99.4, "content_pct": 0.0, "tool_call_pct": 0.6 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 23.077, "latency": 1024.619, "stderr": 2.928, "cost_per_test": 0.682218, "token_totals": { "input_tokens": 341401669, "output_tokens": 11003300, "reasoning_tokens": 8033786, "cache_read_tokens": 300570368, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 32.8, "avg_query_seconds": 34.4, "median_query_seconds": 8.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 20.673, "latency": 242.05, "stderr": 2.815, "cost_per_test": 1.100145, "token_totals": { "input_tokens": 354575717, "output_tokens": 2983139, "reasoning_tokens": 2206660, "cache_read_tokens": 321431187, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 22.8, "avg_query_seconds": 10.5, "median_query_seconds": 6.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 18.269, "latency": 187.7, "stderr": 2.686, "cost_per_test": 0.135395, "token_totals": { "input_tokens": 80643637, "output_tokens": 3487795, "reasoning_tokens": 2539699, "cache_read_tokens": 50273506, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 14.7, "avg_query_seconds": 12.7, "median_query_seconds": 6.9, "avg_ttft_seconds": 2.1, "output_time_split": { "reasoning_pct": 87.9, "content_pct": 11.4, "tool_call_pct": 0.7 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 16.346, "latency": 831.995, "stderr": 2.57, "cost_per_test": 0.295949, "token_totals": { "input_tokens": 182291959, "output_tokens": 7407756, "reasoning_tokens": 4986270, "cache_read_tokens": 118507648, "cache_write_tokens": null }, "token_tasks_measured": 206, "turn_metrics": { "tasks_measured": 206, "avg_turns": 22.5, "avg_query_seconds": 36.9, "median_query_seconds": 12.5, "avg_ttft_seconds": 3.5, "output_time_split": { "reasoning_pct": 49.4, "content_pct": 3.3, "tool_call_pct": 47.3 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 15.865, "latency": 416.437, "stderr": 2.539, "cost_per_test": 0.082648, "token_totals": { "input_tokens": 179797926, "output_tokens": 3117976, "reasoning_tokens": 0, "cache_read_tokens": 147737472, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 24.1, "avg_query_seconds": 17.3, "median_query_seconds": 8.4, "avg_ttft_seconds": 3.8, "output_time_split": { "reasoning_pct": 44.7, "content_pct": 1.5, "tool_call_pct": 53.8 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 15.865, "latency": 1332.514, "stderr": 2.539, "cost_per_test": 0.23035, "token_totals": { "input_tokens": 207645988, "output_tokens": 4488005, "reasoning_tokens": 2618301, "cache_read_tokens": 180277739, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 27.5, "avg_query_seconds": 48.5, "median_query_seconds": 36.8, "avg_ttft_seconds": 23.5, "output_time_split": { "reasoning_pct": 51.3, "content_pct": 12.7, "tool_call_pct": 36.1 } }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 15.865, "latency": 1853.133, "stderr": 2.539, "cost_per_test": 1.520305, "token_totals": { "input_tokens": 1570147541, "output_tokens": 3643156, "reasoning_tokens": 0, "cache_read_tokens": 1492103312, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 92.8, "avg_query_seconds": 20.5, "median_query_seconds": 5.8, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.3": { "accuracy": 15.385, "latency": 84.583, "stderr": 2.508, "cost_per_test": 0.203143, "token_totals": { "input_tokens": 65647893, "output_tokens": 3371863, "reasoning_tokens": 2425742, "cache_read_tokens": 48015040, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 14.4, "avg_query_seconds": 6.4, "median_query_seconds": 3.3, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 15.385, "latency": 515.464, "stderr": 2.508, "cost_per_test": 0.028227, "token_totals": { "input_tokens": 221182940, "output_tokens": 7577859, "reasoning_tokens": null, "cache_read_tokens": 182576832, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 26.5, "avg_query_seconds": 19.5, "median_query_seconds": 8.9, "avg_ttft_seconds": 5.6, "output_time_split": { "reasoning_pct": 59.5, "content_pct": 2.9, "tool_call_pct": 37.6 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 15.385, "latency": 611.366, "stderr": 2.508, "cost_per_test": null, "token_totals": { "input_tokens": 181276135, "output_tokens": 3143670, "reasoning_tokens": null, "cache_read_tokens": 49852416, "cache_write_tokens": null }, "token_tasks_measured": 205, "turn_metrics": { "tasks_measured": 205, "avg_turns": 29.6, "avg_query_seconds": 20.6, "median_query_seconds": 9.4, "avg_ttft_seconds": 7.9, "output_time_split": { "reasoning_pct": 52.8, "content_pct": 14.4, "tool_call_pct": 32.9 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 14.904, "latency": 658.954, "stderr": 2.475, "cost_per_test": 0.697997, "token_totals": { "input_tokens": 228091457, "output_tokens": 5468376, "reasoning_tokens": 3418357, "cache_read_tokens": 0, "cache_write_tokens": null }, "token_tasks_measured": 207, "turn_metrics": { "tasks_measured": 207, "avg_turns": 26.6, "avg_query_seconds": 24.8, "median_query_seconds": 9.9, "avg_ttft_seconds": 4.0, "output_time_split": { "reasoning_pct": 42.7, "content_pct": 1.2, "tool_call_pct": 56.1 } }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 13.942, "latency": 59.197, "stderr": 2.408, "cost_per_test": 0.124723, "token_totals": { "input_tokens": 121402055, "output_tokens": 1885651, "reasoning_tokens": 1270438, "cache_read_tokens": 56268106, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 19.0, "avg_query_seconds": 3.1, "median_query_seconds": 1.9, "avg_ttft_seconds": 1.1, "output_time_split": { "reasoning_pct": 90.3, "content_pct": 8.6, "tool_call_pct": 1.1 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 13.942, "latency": 244.487, "stderr": 2.408, "cost_per_test": 0.261499, "token_totals": { "input_tokens": 45228797, "output_tokens": 1394502, "reasoning_tokens": 691251, "cache_read_tokens": 30303746, "cache_write_tokens": 0 }, "token_tasks_measured": 207, "turn_metrics": { "tasks_measured": 207, "avg_turns": 12.5, "avg_query_seconds": 19.6, "median_query_seconds": 4.0, "avg_ttft_seconds": 5.9, "output_time_split": { "reasoning_pct": 58.2, "content_pct": 2.7, "tool_call_pct": 39.0 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 12.5, "latency": 2256.339, "stderr": 2.299, "cost_per_test": 1.864673, "token_totals": { "input_tokens": 742970115, "output_tokens": 55568745, "reasoning_tokens": 53294787, "cache_read_tokens": 622977536, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 45.0, "avg_query_seconds": 50.1, "median_query_seconds": 8.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 10.577, "latency": 243.38, "stderr": 2.138, "cost_per_test": 0.26891, "token_totals": { "input_tokens": 131779295, "output_tokens": 2744297, "reasoning_tokens": 1091315, "cache_read_tokens": 106353006, "cache_write_tokens": 24601089 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 17.2, "avg_query_seconds": 14.2, "median_query_seconds": 6.8, "avg_ttft_seconds": 1.2, "output_time_split": { "reasoning_pct": 49.2, "content_pct": 16.8, "tool_call_pct": 34.0 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 10.577, "latency": 420.647, "stderr": 2.138, "cost_per_test": 0.219167, "token_totals": { "input_tokens": 143277305, "output_tokens": 5052868, "reasoning_tokens": 3440031, "cache_read_tokens": 14839186, "cache_write_tokens": 7287836 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 26.4, "avg_query_seconds": 15.9, "median_query_seconds": 5.0, "avg_ttft_seconds": 2.1, "output_time_split": { "reasoning_pct": 55.4, "content_pct": 8.9, "tool_call_pct": 35.8 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 10.096, "latency": 82.971, "stderr": 2.094, "cost_per_test": 0.063761, "token_totals": { "input_tokens": 92421794, "output_tokens": 5513825, "reasoning_tokens": 3920900, "cache_read_tokens": 49868032, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 16.2, "avg_query_seconds": 5.1, "median_query_seconds": 2.7, "avg_ttft_seconds": 0.5, "output_time_split": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 9.135, "latency": 391.469, "stderr": 2.002, "cost_per_test": 0.033318, "token_totals": { "input_tokens": 251871878, "output_tokens": 3303008, "reasoning_tokens": 1325334, "cache_read_tokens": 213241152, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 35.4, "avg_query_seconds": 11.1, "median_query_seconds": 6.6, "avg_ttft_seconds": 3.9, "output_time_split": { "reasoning_pct": 52.6, "content_pct": 5.3, "tool_call_pct": 42.1 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 9.135, "latency": 613.248, "stderr": 2.002, "cost_per_test": 1.308079, "token_totals": { "input_tokens": 148782567, "output_tokens": 6520890, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 25.9, "avg_query_seconds": 23.7, "median_query_seconds": 9.4, "avg_ttft_seconds": 6.0, "output_time_split": { "reasoning_pct": 85.8, "content_pct": 12.5, "tool_call_pct": 1.7 } }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 6.25, "latency": 526.838, "stderr": 1.682, "cost_per_test": 0.141253, "token_totals": { "input_tokens": 386062284, "output_tokens": 6536232, "reasoning_tokens": 4445328, "cache_read_tokens": 311122688, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 40.6, "avg_query_seconds": 13.0, "median_query_seconds": 6.7, "avg_ttft_seconds": 3.5, "output_time_split": { "reasoning_pct": 71.7, "content_pct": 0.0, "tool_call_pct": 28.3 } }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 4.327, "latency": 47.609, "stderr": 1.414, "cost_per_test": 0.058112, "token_totals": { "input_tokens": 77109229, "output_tokens": 1908210, "reasoning_tokens": 1539184, "cache_read_tokens": 25000429, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 15.4, "avg_query_seconds": 3.1, "median_query_seconds": 2.0, "avg_ttft_seconds": 1.6, "output_time_split": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 3.365, "latency": 20.403, "stderr": 1.253, "cost_per_test": 0.013607, "token_totals": { "input_tokens": 8137148, "output_tokens": 895676, "reasoning_tokens": 458069, "cache_read_tokens": 2433959, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 4.3, "avg_query_seconds": 4.8, "median_query_seconds": 4.2, "avg_ttft_seconds": 1.4, "output_time_split": { "reasoning_pct": 65.6, "content_pct": 33.7, "tool_call_pct": 0.7 } }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 2.885, "latency": 285.48, "stderr": 1.163, "cost_per_test": 0.004365, "token_totals": { "input_tokens": 24564654, "output_tokens": 1677472, "reasoning_tokens": null, "cache_read_tokens": 16501798, "cache_write_tokens": 0 }, "token_tasks_measured": 207, "turn_metrics": { "tasks_measured": 208, "avg_turns": 9.3, "avg_query_seconds": 30.4, "median_query_seconds": 2.2, "avg_ttft_seconds": 14.5, "output_time_split": { "reasoning_pct": 68.7, "content_pct": 28.4, "tool_call_pct": 2.9 } }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 2.404, "latency": 466.26, "stderr": 1.065, "cost_per_test": 0.219839, "token_totals": { "input_tokens": 388327686, "output_tokens": 3513231, "reasoning_tokens": 1795689, "cache_read_tokens": 333442324, "cache_write_tokens": 0 }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 44.2, "avg_query_seconds": 10.5, "median_query_seconds": 4.7, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.962, "latency": 352.036, "stderr": 0.678, "cost_per_test": 0.107892, "token_totals": { "input_tokens": 363797527, "output_tokens": 2340571, "reasoning_tokens": 1061598, "cache_read_tokens": 288125104, "cache_write_tokens": null }, "token_tasks_measured": 208, "turn_metrics": { "tasks_measured": 208, "avg_turns": 41.7, "avg_query_seconds": 8.4, "median_query_seconds": 3.2, "avg_ttft_seconds": null, "output_time_split": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 211.833, "stderr": 0.0, "cost_per_test": 0.036023, "token_totals": { "input_tokens": 190331294, "output_tokens": 4335412, "reasoning_tokens": null, "cache_read_tokens": 130131626, "cache_write_tokens": null }, "token_tasks_measured": 206, "turn_metrics": { "tasks_measured": 207, "avg_turns": 21.3, "avg_query_seconds": 9.9, "median_query_seconds": 5.3, "avg_ttft_seconds": 2.4, "output_time_split": { "reasoning_pct": 66.4, "content_pct": 2.6, "tool_call_pct": 31.0 } }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null } }, "administrative_regulatory": { "anthropic/claude-sonnet-5-5": { "accuracy": 69.697, "latency": 4690.9, "stderr": 5.7, "cost_per_test": 11.667322, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 68.182, "latency": 1486.33, "stderr": 5.777, "cost_per_test": 4.807402, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 68.182, "latency": 3106.859, "stderr": 5.777, "cost_per_test": 20.305794, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 65.152, "latency": 1532.62, "stderr": 5.91, "cost_per_test": 6.583578, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-carbon": { "accuracy": 65.152, "latency": 2535.332, "stderr": 5.91, "cost_per_test": 6.803146, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 65.152, "latency": 5979.926, "stderr": 5.91, "cost_per_test": 20.862676, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 63.636, "latency": 749.583, "stderr": 5.967, "cost_per_test": 3.444134, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse-wonder": { "accuracy": 63.636, "latency": 889.246, "stderr": 5.967, "cost_per_test": 3.922674, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 63.636, "latency": 1691.752, "stderr": 5.967, "cost_per_test": 3.18787, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "fireworks/ember-1": { "accuracy": 63.636, "latency": 2038.532, "stderr": 5.967, "cost_per_test": 3.174554, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.6": { "accuracy": 62.121, "latency": 1372.59, "stderr": 6.017, "cost_per_test": 1.517644, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 62.121, "latency": 2956.379, "stderr": 6.017, "cost_per_test": 2.133253, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 60.606, "latency": 306.876, "stderr": 6.061, "cost_per_test": 0.540035, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 60.606, "latency": 631.256, "stderr": 6.061, "cost_per_test": 0.243078, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 60.606, "latency": 4138.255, "stderr": 6.061, "cost_per_test": 19.689902, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 59.091, "latency": 1609.837, "stderr": 6.098, "cost_per_test": 0.17954, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 59.091, "latency": 1931.905, "stderr": 6.098, "cost_per_test": 0.652538, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 59.091, "latency": 3192.985, "stderr": 6.098, "cost_per_test": 2.201925, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 57.576, "latency": 1179.378, "stderr": 6.13, "cost_per_test": 9.381311, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 57.576, "latency": 2848.785, "stderr": 6.13, "cost_per_test": 0.198792, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 57.576, "latency": 3607.895, "stderr": 6.13, "cost_per_test": 6.866314, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 56.061, "latency": 5673.421, "stderr": 6.156, "cost_per_test": 6.338985, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.7": { "accuracy": 54.545, "latency": 1016.365, "stderr": 6.176, "cost_per_test": 4.175009, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 54.545, "latency": 1285.722, "stderr": 6.176, "cost_per_test": 2.507811, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 54.545, "latency": 3713.178, "stderr": 6.176, "cost_per_test": 0.834838, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 53.03, "latency": 656.731, "stderr": 6.19, "cost_per_test": 0.246492, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 53.03, "latency": 1103.075, "stderr": 6.19, "cost_per_test": 8.442858, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 53.03, "latency": 1439.965, "stderr": 6.19, "cost_per_test": 2.756725, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 53.03, "latency": 2093.383, "stderr": 6.19, "cost_per_test": 7.953883, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 51.515, "latency": 481.922, "stderr": 6.199, "cost_per_test": 0.624599, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.5": { "accuracy": 51.515, "latency": 1001.46, "stderr": 6.199, "cost_per_test": 1.36196, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "tencent/hy4": { "accuracy": 51.515, "latency": 5301.951, "stderr": 6.199, "cost_per_test": 2.211937, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse-spark-open": { "accuracy": 50.0, "latency": 1671.25, "stderr": 6.202, "cost_per_test": 1.423111, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 50.0, "latency": 2424.284, "stderr": 6.202, "cost_per_test": 1.073701, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 50.0, "latency": 3941.13, "stderr": 6.202, "cost_per_test": 1.441703, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 50.0, "latency": 7602.624, "stderr": 6.202, "cost_per_test": 5.772668, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 48.485, "latency": 302.672, "stderr": 6.199, "cost_per_test": 0.506511, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "stepfun/step-5-preview": { "accuracy": 48.485, "latency": 2171.348, "stderr": 6.199, "cost_per_test": 0.862352, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 48.485, "latency": 2245.491, "stderr": 6.199, "cost_per_test": 0.755621, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 46.97, "latency": 296.369, "stderr": 6.19, "cost_per_test": 1.661721, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 46.97, "latency": 715.742, "stderr": 6.19, "cost_per_test": 0.066967, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 46.97, "latency": 1523.131, "stderr": 6.19, "cost_per_test": 0.921373, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-legal": { "accuracy": 46.97, "latency": 2358.521, "stderr": 6.19, "cost_per_test": 0.049642, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 45.455, "latency": 425.156, "stderr": 6.176, "cost_per_test": 3.06182, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 45.455, "latency": 1643.295, "stderr": 6.176, "cost_per_test": 2.2119, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 45.455, "latency": 3381.142, "stderr": 6.176, "cost_per_test": 2.810214, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 43.939, "latency": 285.017, "stderr": 6.156, "cost_per_test": 1.356228, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 43.939, "latency": 636.543, "stderr": 6.156, "cost_per_test": 2.731524, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 43.939, "latency": 901.446, "stderr": 6.156, "cost_per_test": 2.410326, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 42.424, "latency": 94.03, "stderr": 6.13, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 42.424, "latency": 244.664, "stderr": 6.13, "cost_per_test": 0.704178, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 40.909, "latency": 2966.067, "stderr": 6.098, "cost_per_test": 0.472211, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "zai/glm-5.2": { "accuracy": 39.394, "latency": 836.031, "stderr": 6.061, "cost_per_test": 0.808378, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 39.394, "latency": 1112.655, "stderr": 6.061, "cost_per_test": 3.887103, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 37.879, "latency": 282.392, "stderr": 6.017, "cost_per_test": 0.382532, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-luna": { "accuracy": 37.879, "latency": 2341.023, "stderr": 6.017, "cost_per_test": 0.409481, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 36.364, "latency": 755.036, "stderr": 5.967, "cost_per_test": 0.352926, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 36.364, "latency": 789.443, "stderr": 5.967, "cost_per_test": 1.359392, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 36.364, "latency": 1185.766, "stderr": 5.967, "cost_per_test": 0.295871, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "inception/mercury-3": { "accuracy": 34.848, "latency": 434.539, "stderr": 5.91, "cost_per_test": 0.075638, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 34.848, "latency": 5582.268, "stderr": 5.91, "cost_per_test": 2.616141, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 31.818, "latency": 139.396, "stderr": 5.777, "cost_per_test": 0.409841, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 31.818, "latency": 307.232, "stderr": 5.777, "cost_per_test": 0.323431, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 31.818, "latency": 877.92, "stderr": 5.777, "cost_per_test": 0.44143, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 28.788, "latency": 842.128, "stderr": 5.616, "cost_per_test": 0.134944, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 28.788, "latency": 903.624, "stderr": 5.616, "cost_per_test": 0.671217, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 27.273, "latency": 166.672, "stderr": 5.524, "cost_per_test": 0.129186, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 25.758, "latency": 72.945, "stderr": 5.424, "cost_per_test": 0.19233, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 24.242, "latency": 219.629, "stderr": 5.316, "cost_per_test": 1.225685, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 19.697, "latency": 50.778, "stderr": 4.933, "cost_per_test": 0.123387, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 19.697, "latency": 625.818, "stderr": 4.933, "cost_per_test": 0.670062, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 18.182, "latency": 381.12, "stderr": 4.784, "cost_per_test": 0.087857, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 18.182, "latency": 494.675, "stderr": 4.784, "cost_per_test": 0.027887, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 18.182, "latency": 1689.904, "stderr": 4.784, "cost_per_test": 1.488125, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 16.667, "latency": 382.835, "stderr": 4.623, "cost_per_test": 0.200329, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 16.667, "latency": 476.589, "stderr": 4.623, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 16.667, "latency": 1153.879, "stderr": 4.623, "cost_per_test": 0.195218, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 15.152, "latency": 805.914, "stderr": 4.447, "cost_per_test": 0.359357, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 15.152, "latency": 1961.008, "stderr": 4.447, "cost_per_test": 1.456649, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 10.606, "latency": 199.066, "stderr": 3.819, "cost_per_test": 0.274657, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 10.606, "latency": 343.766, "stderr": 3.819, "cost_per_test": 0.038807, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 10.606, "latency": 402.197, "stderr": 3.819, "cost_per_test": 0.125736, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 9.091, "latency": 226.406, "stderr": 3.566, "cost_per_test": 0.379263, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 7.576, "latency": 575.292, "stderr": 3.282, "cost_per_test": 1.130658, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 6.061, "latency": 69.443, "stderr": 2.96, "cost_per_test": 0.05243, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "inception/mercury-2.5": { "accuracy": 4.545, "latency": 41.907, "stderr": 2.584, "cost_per_test": 0.046871, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 3.03, "latency": 18.892, "stderr": 2.126, "cost_per_test": 0.013517, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 3.03, "latency": 383.823, "stderr": 2.126, "cost_per_test": 0.163947, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 1.515, "latency": 332.137, "stderr": 1.515, "cost_per_test": 0.087793, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 1.515, "latency": 409.746, "stderr": 1.515, "cost_per_test": 0.004424, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 215.774, "stderr": 0.0, "cost_per_test": 0.032184, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null } }, "business_commercial": { "anthropic/claude-opus-5": { "accuracy": 62.791, "latency": 2163.785, "stderr": 7.458, "cost_per_test": 7.357666, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 62.791, "latency": 3855.165, "stderr": 7.458, "cost_per_test": 27.230592, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 58.14, "latency": 353.92, "stderr": 7.612, "cost_per_test": 0.626202, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 58.14, "latency": 1748.44, "stderr": 7.612, "cost_per_test": 3.900906, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 53.488, "latency": 4581.547, "stderr": 7.696, "cost_per_test": 19.023356, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 51.163, "latency": 319.151, "stderr": 7.713, "cost_per_test": 0.363621, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 51.163, "latency": 1482.098, "stderr": 7.713, "cost_per_test": 9.515091, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 48.837, "latency": 904.491, "stderr": 7.713, "cost_per_test": 3.808111, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 48.837, "latency": 1612.05, "stderr": 7.713, "cost_per_test": 1.54459, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 48.837, "latency": 7461.121, "stderr": 7.713, "cost_per_test": 26.294195, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 46.512, "latency": 755.219, "stderr": 7.696, "cost_per_test": 0.250157, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-4-argon": { "accuracy": 46.512, "latency": 2252.454, "stderr": 7.696, "cost_per_test": 7.818685, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3": { "accuracy": 46.512, "latency": 3261.478, "stderr": 7.696, "cost_per_test": 2.29039, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-4-carbon": { "accuracy": 46.512, "latency": 3764.53, "stderr": 7.696, "cost_per_test": 10.672138, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 46.512, "latency": 3920.461, "stderr": 7.696, "cost_per_test": 2.547702, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 46.512, "latency": 8276.057, "stderr": 7.696, "cost_per_test": 5.198048, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 44.186, "latency": 328.596, "stderr": 7.663, "cost_per_test": 0.506176, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 44.186, "latency": 1380.854, "stderr": 7.663, "cost_per_test": 4.555398, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 44.186, "latency": 1783.688, "stderr": 7.663, "cost_per_test": 0.170401, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 44.186, "latency": 2247.75, "stderr": 7.663, "cost_per_test": 0.705293, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 44.186, "latency": 3410.204, "stderr": 7.663, "cost_per_test": 0.266594, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 44.186, "latency": 4069.494, "stderr": 7.663, "cost_per_test": 0.863275, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 44.186, "latency": 5681.673, "stderr": 7.663, "cost_per_test": 1.820007, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 41.86, "latency": 315.567, "stderr": 7.612, "cost_per_test": 1.733887, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 41.86, "latency": 777.706, "stderr": 7.612, "cost_per_test": 2.888428, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 41.86, "latency": 1723.973, "stderr": 7.612, "cost_per_test": 2.311817, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 39.535, "latency": 694.588, "stderr": 7.544, "cost_per_test": 0.226487, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 39.535, "latency": 790.908, "stderr": 7.544, "cost_per_test": 3.239177, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 39.535, "latency": 1523.334, "stderr": 7.544, "cost_per_test": 1.137443, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 39.535, "latency": 2344.543, "stderr": 7.544, "cost_per_test": 0.811263, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 39.535, "latency": 2357.851, "stderr": 7.544, "cost_per_test": 3.455558, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4": { "accuracy": 39.535, "latency": 5721.537, "stderr": 7.544, "cost_per_test": 2.262179, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 39.535, "latency": 6273.67, "stderr": 7.544, "cost_per_test": 15.137463, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 37.209, "latency": 298.916, "stderr": 7.458, "cost_per_test": 0.773586, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 37.209, "latency": 909.412, "stderr": 7.458, "cost_per_test": 0.066442, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.5": { "accuracy": 37.209, "latency": 2266.219, "stderr": 7.458, "cost_per_test": 6.684872, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-legal": { "accuracy": 37.209, "latency": 2576.157, "stderr": 7.458, "cost_per_test": 0.05022, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 37.209, "latency": 2633.008, "stderr": 7.458, "cost_per_test": 0.372136, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "fireworks/ember-1": { "accuracy": 37.209, "latency": 2895.709, "stderr": 7.458, "cost_per_test": 4.184362, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 37.209, "latency": 4084.965, "stderr": 7.458, "cost_per_test": 7.302701, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 34.884, "latency": 709.804, "stderr": 7.354, "cost_per_test": 0.132473, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 34.884, "latency": 1023.821, "stderr": 7.354, "cost_per_test": 2.037941, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 34.884, "latency": 2499.931, "stderr": 7.354, "cost_per_test": 1.025341, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 32.558, "latency": 581.128, "stderr": 7.231, "cost_per_test": 0.642459, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 32.558, "latency": 626.261, "stderr": 7.231, "cost_per_test": 3.528557, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 32.558, "latency": 1735.645, "stderr": 7.231, "cost_per_test": 10.624987, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 32.558, "latency": 1790.227, "stderr": 7.231, "cost_per_test": 3.031928, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 32.558, "latency": 2334.81, "stderr": 7.231, "cost_per_test": 0.78052, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 30.233, "latency": 913.577, "stderr": 7.087, "cost_per_test": 0.893714, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 30.233, "latency": 1961.481, "stderr": 7.087, "cost_per_test": 1.153979, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 27.907, "latency": 317.652, "stderr": 6.921, "cost_per_test": 1.232191, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.2": { "accuracy": 27.907, "latency": 1106.364, "stderr": 6.921, "cost_per_test": 0.920113, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 27.907, "latency": 1674.291, "stderr": 6.921, "cost_per_test": 5.299121, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 27.907, "latency": 6424.655, "stderr": 6.921, "cost_per_test": 7.199044, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 25.581, "latency": 108.067, "stderr": 6.733, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "inception/mercury-3": { "accuracy": 25.581, "latency": 498.212, "stderr": 6.733, "cost_per_test": 0.086143, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "thinkingmachines/inkling": { "accuracy": 25.581, "latency": 792.476, "stderr": 6.733, "cost_per_test": 0.529635, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 25.581, "latency": 867.939, "stderr": 6.733, "cost_per_test": 0.34538, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 25.581, "latency": 5387.125, "stderr": 6.733, "cost_per_test": 1.920252, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 23.256, "latency": 173.23, "stderr": 6.519, "cost_per_test": 0.487431, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 23.256, "latency": 216.623, "stderr": 6.519, "cost_per_test": 0.45295, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 23.256, "latency": 336.248, "stderr": 6.519, "cost_per_test": 0.399162, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 23.256, "latency": 917.178, "stderr": 6.519, "cost_per_test": 0.622362, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 23.256, "latency": 1087.928, "stderr": 6.519, "cost_per_test": 0.215579, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 23.256, "latency": 2678.679, "stderr": 6.519, "cost_per_test": 0.455548, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 20.93, "latency": 913.289, "stderr": 6.277, "cost_per_test": 1.493846, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 20.93, "latency": 1873.65, "stderr": 6.277, "cost_per_test": 1.432344, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 18.605, "latency": 200.727, "stderr": 6.005, "cost_per_test": 0.135638, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 16.279, "latency": 241.456, "stderr": 5.696, "cost_per_test": 0.168225, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 16.279, "latency": 663.553, "stderr": 5.696, "cost_per_test": 0.73626, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 16.279, "latency": 5207.222, "stderr": 5.696, "cost_per_test": 3.545917, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 13.953, "latency": 73.592, "stderr": 5.347, "cost_per_test": 0.170453, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 13.953, "latency": 343.417, "stderr": 5.347, "cost_per_test": 0.211419, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 13.953, "latency": 489.906, "stderr": 5.347, "cost_per_test": 0.032371, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 13.953, "latency": 787.349, "stderr": 5.347, "cost_per_test": 0.216443, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 13.953, "latency": 1493.936, "stderr": 5.347, "cost_per_test": 0.245468, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 11.628, "latency": 44.657, "stderr": 4.946, "cost_per_test": 0.057037, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 11.628, "latency": 108.913, "stderr": 4.946, "cost_per_test": 0.08423, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 11.628, "latency": 452.214, "stderr": 4.946, "cost_per_test": 0.075902, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 11.628, "latency": 620.168, "stderr": 4.946, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 11.628, "latency": 2506.739, "stderr": 4.946, "cost_per_test": 2.128528, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 9.302, "latency": 279.601, "stderr": 4.482, "cost_per_test": 0.268062, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 9.302, "latency": 405.023, "stderr": 4.482, "cost_per_test": 0.029283, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 9.302, "latency": 637.024, "stderr": 4.482, "cost_per_test": 1.423124, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 6.977, "latency": 62.398, "stderr": 3.931, "cost_per_test": 0.120286, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 6.977, "latency": 568.266, "stderr": 3.931, "cost_per_test": 0.150807, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 4.651, "latency": 124.79, "stderr": 3.249, "cost_per_test": 0.004634, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 4.651, "latency": 402.376, "stderr": 3.249, "cost_per_test": 0.204493, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 2.326, "latency": 23.436, "stderr": 2.326, "cost_per_test": 0.014473, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 261.305, "stderr": 0.0, "cost_per_test": 0.042315, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 399.428, "stderr": 0.0, "cost_per_test": 0.12794, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } }, "civil_litigation": { "meta/muse_spark_1_3_max": { "accuracy": 62.963, "latency": 305.909, "stderr": 9.471, "cost_per_test": 0.581163, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 62.963, "latency": 2079.281, "stderr": 9.471, "cost_per_test": 7.304788, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 62.963, "latency": 3586.008, "stderr": 9.471, "cost_per_test": 25.254544, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 59.259, "latency": 3900.655, "stderr": 9.636, "cost_per_test": 0.854674, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3": { "accuracy": 55.556, "latency": 2589.489, "stderr": 9.745, "cost_per_test": 2.051403, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 51.852, "latency": 744.696, "stderr": 9.799, "cost_per_test": 0.267966, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 51.852, "latency": 806.663, "stderr": 9.799, "cost_per_test": 2.997652, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 51.852, "latency": 1965.433, "stderr": 9.799, "cost_per_test": 6.770674, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 48.148, "latency": 308.847, "stderr": 9.799, "cost_per_test": 0.367043, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 48.148, "latency": 663.732, "stderr": 9.799, "cost_per_test": 0.239442, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "stepfun/step-5-preview": { "accuracy": 48.148, "latency": 2136.958, "stderr": 9.799, "cost_per_test": 0.917209, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "google/gemini-4-carbon": { "accuracy": 48.148, "latency": 3262.96, "stderr": 9.799, "cost_per_test": 9.405202, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 48.148, "latency": 3478.586, "stderr": 9.799, "cost_per_test": 2.299147, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "tencent/hy4": { "accuracy": 48.148, "latency": 5187.901, "stderr": 9.799, "cost_per_test": 2.080453, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 48.148, "latency": 6231.017, "stderr": 9.799, "cost_per_test": 15.071116, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 48.148, "latency": 6638.643, "stderr": 9.799, "cost_per_test": 22.569939, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 48.148, "latency": 7758.778, "stderr": 9.799, "cost_per_test": 5.531551, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k3": { "accuracy": 44.444, "latency": 838.559, "stderr": 9.745, "cost_per_test": 4.229889, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 44.444, "latency": 1104.503, "stderr": 9.745, "cost_per_test": 0.246003, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.7": { "accuracy": 44.444, "latency": 1439.49, "stderr": 9.745, "cost_per_test": 5.606604, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 44.444, "latency": 1439.733, "stderr": 9.745, "cost_per_test": 3.415131, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 44.444, "latency": 1489.886, "stderr": 9.745, "cost_per_test": 10.123039, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 44.444, "latency": 1956.425, "stderr": 9.745, "cost_per_test": 3.400541, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 44.444, "latency": 2408.394, "stderr": 9.745, "cost_per_test": 7.00366, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-legal": { "accuracy": 44.444, "latency": 2608.688, "stderr": 9.745, "cost_per_test": 0.055323, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 40.741, "latency": 557.173, "stderr": 9.636, "cost_per_test": 0.757486, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-astra": { "accuracy": 40.741, "latency": 1743.79, "stderr": 9.636, "cost_per_test": 13.735474, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 40.741, "latency": 2240.6, "stderr": 9.636, "cost_per_test": 0.207888, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 40.741, "latency": 2280.787, "stderr": 9.636, "cost_per_test": 1.099633, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 40.741, "latency": 2417.44, "stderr": 9.636, "cost_per_test": 0.338953, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "fireworks/ember-1": { "accuracy": 40.741, "latency": 2689.198, "stderr": 9.636, "cost_per_test": 4.546668, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 40.741, "latency": 3261.102, "stderr": 9.636, "cost_per_test": 0.259741, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 40.741, "latency": 3816.903, "stderr": 9.636, "cost_per_test": 18.560919, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 40.741, "latency": 5529.804, "stderr": 9.636, "cost_per_test": 1.750979, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 37.037, "latency": 304.101, "stderr": 9.471, "cost_per_test": 0.602276, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 37.037, "latency": 538.692, "stderr": 9.471, "cost_per_test": 2.902212, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 37.037, "latency": 885.474, "stderr": 9.471, "cost_per_test": 4.274558, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 37.037, "latency": 1539.561, "stderr": 9.471, "cost_per_test": 1.681204, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 37.037, "latency": 1554.157, "stderr": 9.471, "cost_per_test": 1.138521, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 37.037, "latency": 2088.431, "stderr": 9.471, "cost_per_test": 0.658074, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "thinkingmachines/inkling": { "accuracy": 33.333, "latency": 740.554, "stderr": 9.245, "cost_per_test": 0.559142, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 33.333, "latency": 1095.538, "stderr": 9.245, "cost_per_test": 0.736017, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.5": { "accuracy": 33.333, "latency": 1225.988, "stderr": 9.245, "cost_per_test": 1.215179, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 33.333, "latency": 5842.182, "stderr": 9.245, "cost_per_test": 2.843836, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 29.63, "latency": 254.44, "stderr": 8.955, "cost_per_test": 1.35934, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 29.63, "latency": 288.591, "stderr": 8.955, "cost_per_test": 0.719234, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-3": { "accuracy": 29.63, "latency": 493.563, "stderr": 8.955, "cost_per_test": 0.076555, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 29.63, "latency": 840.078, "stderr": 8.955, "cost_per_test": 0.059764, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 29.63, "latency": 1001.064, "stderr": 8.955, "cost_per_test": 2.550366, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 29.63, "latency": 2447.426, "stderr": 8.955, "cost_per_test": 3.801126, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 29.63, "latency": 3932.081, "stderr": 8.955, "cost_per_test": 7.563297, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 29.63, "latency": 6255.777, "stderr": 8.955, "cost_per_test": 7.220094, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 25.926, "latency": 246.451, "stderr": 8.594, "cost_per_test": 0.200307, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 25.926, "latency": 775.441, "stderr": 8.594, "cost_per_test": 0.228401, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 25.926, "latency": 823.116, "stderr": 8.594, "cost_per_test": 0.385069, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 25.926, "latency": 876.732, "stderr": 8.594, "cost_per_test": 1.504309, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.2": { "accuracy": 25.926, "latency": 885.063, "stderr": 8.594, "cost_per_test": 0.857222, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 25.926, "latency": 1273.914, "stderr": 8.594, "cost_per_test": 1.590079, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 25.926, "latency": 1304.611, "stderr": 8.594, "cost_per_test": 1.737788, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 25.926, "latency": 2149.371, "stderr": 8.594, "cost_per_test": 0.78097, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 25.926, "latency": 2593.828, "stderr": 8.594, "cost_per_test": 0.46119, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 22.222, "latency": 92.542, "stderr": 8.153, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 22.222, "latency": 255.111, "stderr": 8.153, "cost_per_test": 1.230964, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 22.222, "latency": 361.704, "stderr": 8.153, "cost_per_test": 0.473442, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 22.222, "latency": 478.114, "stderr": 8.153, "cost_per_test": 0.031614, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 22.222, "latency": 676.765, "stderr": 8.153, "cost_per_test": 0.13253, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 22.222, "latency": 690.614, "stderr": 8.153, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 18.519, "latency": 89.721, "stderr": 7.618, "cost_per_test": 0.06691, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 18.519, "latency": 192.418, "stderr": 7.618, "cost_per_test": 0.477549, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 18.519, "latency": 1372.833, "stderr": 7.618, "cost_per_test": 0.220822, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 18.519, "latency": 1441.302, "stderr": 7.618, "cost_per_test": 5.211283, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 18.519, "latency": 1733.848, "stderr": 7.618, "cost_per_test": 1.440762, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 14.815, "latency": 428.141, "stderr": 6.967, "cost_per_test": 0.242453, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 14.815, "latency": 698.535, "stderr": 6.967, "cost_per_test": 0.632609, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 14.815, "latency": 2120.497, "stderr": 6.967, "cost_per_test": 1.875076, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 14.815, "latency": 4095.85, "stderr": 6.967, "cost_per_test": 3.018691, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 11.111, "latency": 56.454, "stderr": 6.163, "cost_per_test": 0.108873, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 11.111, "latency": 215.912, "stderr": 6.163, "cost_per_test": 0.132078, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 11.111, "latency": 225.619, "stderr": 6.163, "cost_per_test": 0.508937, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 11.111, "latency": 416.27, "stderr": 6.163, "cost_per_test": 0.037073, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 11.111, "latency": 486.481, "stderr": 6.163, "cost_per_test": 0.080617, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 11.111, "latency": 595.409, "stderr": 6.163, "cost_per_test": 1.316462, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 7.407, "latency": 277.344, "stderr": 5.136, "cost_per_test": 0.287416, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 7.407, "latency": 725.151, "stderr": 5.136, "cost_per_test": 0.180106, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 3.704, "latency": 19.793, "stderr": 3.704, "cost_per_test": 0.013273, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 3.704, "latency": 47.033, "stderr": 3.704, "cost_per_test": 0.064712, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "grok/grok-4.3": { "accuracy": 3.704, "latency": 68.656, "stderr": 3.704, "cost_per_test": 0.19461, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 3.704, "latency": 273.794, "stderr": 3.704, "cost_per_test": 0.004442, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 3.704, "latency": 332.669, "stderr": 3.704, "cost_per_test": 0.119391, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 186.413, "stderr": 0.0, "cost_per_test": 0.040156, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 450.642, "stderr": 0.0, "cost_per_test": 0.229555, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } }, "constitutional_civil_rights": { "google/gemini-4-carbon": { "accuracy": 55.263, "latency": 2436.37, "stderr": 8.174, "cost_per_test": 6.991706, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 50.0, "latency": 261.358, "stderr": 8.22, "cost_per_test": 0.485181, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 50.0, "latency": 977.356, "stderr": 8.22, "cost_per_test": 6.898245, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 47.368, "latency": 1151.764, "stderr": 8.209, "cost_per_test": 2.721277, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 47.368, "latency": 1361.337, "stderr": 8.209, "cost_per_test": 4.353414, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 47.368, "latency": 1681.88, "stderr": 8.209, "cost_per_test": 0.17967, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 44.737, "latency": 427.276, "stderr": 8.174, "cost_per_test": 2.058545, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 44.737, "latency": 1566.47, "stderr": 8.174, "cost_per_test": 2.477637, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 44.737, "latency": 1835.592, "stderr": 8.174, "cost_per_test": 0.898996, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 44.737, "latency": 1898.737, "stderr": 8.174, "cost_per_test": 6.396226, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 44.737, "latency": 2446.246, "stderr": 8.174, "cost_per_test": 0.179145, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 44.737, "latency": 5507.901, "stderr": 8.174, "cost_per_test": 1.55544, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 44.737, "latency": 6953.006, "stderr": 8.174, "cost_per_test": 4.449226, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 42.105, "latency": 220.844, "stderr": 8.117, "cost_per_test": 1.043129, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 42.105, "latency": 1049.066, "stderr": 8.117, "cost_per_test": 4.00044, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 42.105, "latency": 1675.051, "stderr": 8.117, "cost_per_test": 0.53415, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "fireworks/ember-1": { "accuracy": 42.105, "latency": 1748.11, "stderr": 8.117, "cost_per_test": 2.722205, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 42.105, "latency": 3510.897, "stderr": 8.117, "cost_per_test": 21.64278, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 42.105, "latency": 5827.977, "stderr": 8.117, "cost_per_test": 14.117789, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 42.105, "latency": 6438.62, "stderr": 8.117, "cost_per_test": 22.403219, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 39.474, "latency": 275.608, "stderr": 8.036, "cost_per_test": 0.430216, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 39.474, "latency": 609.116, "stderr": 8.036, "cost_per_test": 2.462184, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 39.474, "latency": 632.775, "stderr": 8.036, "cost_per_test": 2.959097, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 39.474, "latency": 645.429, "stderr": 8.036, "cost_per_test": 0.225678, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 39.474, "latency": 1943.878, "stderr": 8.036, "cost_per_test": 0.828943, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-legal": { "accuracy": 39.474, "latency": 2063.284, "stderr": 8.036, "cost_per_test": 0.037714, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3": { "accuracy": 39.474, "latency": 2583.919, "stderr": 8.036, "cost_per_test": 1.779991, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 39.474, "latency": 2858.564, "stderr": 8.036, "cost_per_test": 0.671944, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 36.842, "latency": 518.106, "stderr": 7.93, "cost_per_test": 0.191689, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 36.842, "latency": 572.241, "stderr": 7.93, "cost_per_test": 2.490256, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.6": { "accuracy": 36.842, "latency": 1396.958, "stderr": 7.93, "cost_per_test": 1.277781, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 36.842, "latency": 1461.512, "stderr": 7.93, "cost_per_test": 0.943808, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "inception/mercury-3": { "accuracy": 34.211, "latency": 353.782, "stderr": 7.799, "cost_per_test": 0.052994, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "zai/glm-5.1": { "accuracy": 34.211, "latency": 355.395, "stderr": 7.799, "cost_per_test": 0.362114, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 34.211, "latency": 522.986, "stderr": 7.799, "cost_per_test": 0.545149, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 34.211, "latency": 733.352, "stderr": 7.799, "cost_per_test": 0.291222, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 34.211, "latency": 830.725, "stderr": 7.799, "cost_per_test": 0.076515, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 34.211, "latency": 853.113, "stderr": 7.799, "cost_per_test": 1.965994, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 34.211, "latency": 1718.661, "stderr": 7.799, "cost_per_test": 0.583776, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 34.211, "latency": 1785.016, "stderr": 7.799, "cost_per_test": 0.721618, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 34.211, "latency": 3263.997, "stderr": 7.799, "cost_per_test": 2.246783, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 34.211, "latency": 3655.679, "stderr": 7.799, "cost_per_test": 6.783177, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 34.211, "latency": 3868.674, "stderr": 7.799, "cost_per_test": 14.526185, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4": { "accuracy": 34.211, "latency": 4486.623, "stderr": 7.799, "cost_per_test": 1.742342, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 34.211, "latency": 4751.209, "stderr": 7.799, "cost_per_test": 2.624566, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 31.579, "latency": 62.09, "stderr": 7.642, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 31.579, "latency": 214.833, "stderr": 7.642, "cost_per_test": 0.667444, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling": { "accuracy": 31.579, "latency": 611.367, "stderr": 7.642, "cost_per_test": 0.372278, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.5": { "accuracy": 31.579, "latency": 907.475, "stderr": 7.642, "cost_per_test": 0.836603, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 31.579, "latency": 1025.06, "stderr": 7.642, "cost_per_test": 0.204391, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 31.579, "latency": 1775.958, "stderr": 7.642, "cost_per_test": 2.692913, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 31.579, "latency": 1958.935, "stderr": 7.642, "cost_per_test": 0.364867, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 31.579, "latency": 6612.654, "stderr": 7.642, "cost_per_test": 7.25464, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 28.947, "latency": 224.844, "stderr": 7.456, "cost_per_test": 0.987781, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 28.947, "latency": 310.439, "stderr": 7.456, "cost_per_test": 0.303515, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.2": { "accuracy": 28.947, "latency": 558.667, "stderr": 7.456, "cost_per_test": 0.395873, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.5": { "accuracy": 28.947, "latency": 2156.642, "stderr": 7.456, "cost_per_test": 5.731692, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 28.947, "latency": 2834.861, "stderr": 7.456, "cost_per_test": 0.421978, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-astra": { "accuracy": 26.316, "latency": 1280.117, "stderr": 7.239, "cost_per_test": 8.37428, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 26.316, "latency": 1286.448, "stderr": 7.239, "cost_per_test": 1.563366, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 23.684, "latency": 782.498, "stderr": 6.989, "cost_per_test": 1.136745, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 23.684, "latency": 3732.629, "stderr": 6.989, "cost_per_test": 2.673236, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 21.053, "latency": 61.408, "stderr": 6.702, "cost_per_test": 0.080105, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 21.053, "latency": 223.214, "stderr": 6.702, "cost_per_test": 0.466165, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 21.053, "latency": 828.518, "stderr": 6.702, "cost_per_test": 0.208128, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 21.053, "latency": 1564.212, "stderr": 6.702, "cost_per_test": 1.25835, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 18.421, "latency": 160.664, "stderr": 6.373, "cost_per_test": 0.393807, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 18.421, "latency": 279.901, "stderr": 6.373, "cost_per_test": 0.255064, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 18.421, "latency": 1202.973, "stderr": 6.373, "cost_per_test": 4.442219, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 15.789, "latency": 1326.72, "stderr": 5.995, "cost_per_test": 0.216972, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 13.158, "latency": 74.98, "stderr": 5.557, "cost_per_test": 0.047883, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "grok/grok-4.3": { "accuracy": 13.158, "latency": 125.212, "stderr": 5.557, "cost_per_test": 0.250093, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 13.158, "latency": 276.602, "stderr": 5.557, "cost_per_test": 0.260695, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 13.158, "latency": 384.09, "stderr": 5.557, "cost_per_test": 0.079592, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 13.158, "latency": 822.903, "stderr": 5.557, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 13.158, "latency": 1096.62, "stderr": 5.557, "cost_per_test": 0.614384, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 13.158, "latency": 1433.795, "stderr": 5.557, "cost_per_test": 1.467472, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 10.526, "latency": 303.387, "stderr": 5.045, "cost_per_test": 0.019445, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 10.526, "latency": 449.168, "stderr": 5.045, "cost_per_test": 0.209051, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 10.526, "latency": 589.79, "stderr": 5.045, "cost_per_test": 0.111862, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 10.526, "latency": 696.299, "stderr": 5.045, "cost_per_test": 0.516518, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 10.526, "latency": 715.501, "stderr": 5.045, "cost_per_test": 1.425655, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 7.895, "latency": 214.461, "stderr": 4.433, "cost_per_test": 0.119798, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 7.895, "latency": 352.653, "stderr": 4.433, "cost_per_test": 0.029482, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 5.263, "latency": 268.813, "stderr": 3.671, "cost_per_test": 0.003527, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 2.632, "latency": 21.603, "stderr": 2.632, "cost_per_test": 0.013343, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 2.632, "latency": 53.037, "stderr": 2.632, "cost_per_test": 0.058844, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-m.1": { "accuracy": 2.632, "latency": 540.649, "stderr": 2.632, "cost_per_test": 0.230889, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 2.632, "latency": 597.509, "stderr": 2.632, "cost_per_test": 0.134636, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 176.967, "stderr": 0.0, "cost_per_test": 0.030415, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 293.416, "stderr": 0.0, "cost_per_test": 0.090182, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } }, "criminal": { "google/gemini-4-carbon": { "accuracy": 59.649, "latency": 3164.955, "stderr": 6.556, "cost_per_test": 8.842662, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-4-argon": { "accuracy": 56.14, "latency": 1907.88, "stderr": 6.631, "cost_per_test": 6.616583, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 49.123, "latency": 1527.247, "stderr": 6.681, "cost_per_test": 3.617802, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 47.368, "latency": 303.011, "stderr": 6.672, "cost_per_test": 0.60538, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 47.368, "latency": 1371.988, "stderr": 6.672, "cost_per_test": 5.268708, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 45.614, "latency": 755.757, "stderr": 6.656, "cost_per_test": 2.852274, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 43.86, "latency": 967.654, "stderr": 6.631, "cost_per_test": 4.059119, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 43.86, "latency": 1928.013, "stderr": 6.631, "cost_per_test": 0.177786, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 43.86, "latency": 2118.512, "stderr": 6.631, "cost_per_test": 6.903486, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 43.86, "latency": 4987.986, "stderr": 6.631, "cost_per_test": 24.12378, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 43.86, "latency": 7534.426, "stderr": 6.631, "cost_per_test": 5.274524, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 42.105, "latency": 1356.005, "stderr": 6.598, "cost_per_test": 1.460147, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 42.105, "latency": 1398.537, "stderr": 6.598, "cost_per_test": 9.468125, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 42.105, "latency": 5566.578, "stderr": 6.598, "cost_per_test": 1.725041, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 42.105, "latency": 7162.351, "stderr": 6.598, "cost_per_test": 25.272629, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 40.351, "latency": 315.871, "stderr": 6.556, "cost_per_test": 1.534611, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 40.351, "latency": 1506.644, "stderr": 6.556, "cost_per_test": 2.518676, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 40.351, "latency": 3997.963, "stderr": 6.556, "cost_per_test": 2.676114, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 38.596, "latency": 301.211, "stderr": 6.505, "cost_per_test": 0.446419, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "stepfun/step-5-preview": { "accuracy": 38.596, "latency": 2210.107, "stderr": 6.505, "cost_per_test": 0.87429, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "zai/glm-5.3": { "accuracy": 38.596, "latency": 2957.873, "stderr": 6.505, "cost_per_test": 2.182135, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 38.596, "latency": 3466.552, "stderr": 6.505, "cost_per_test": 22.326482, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 38.596, "latency": 6259.782, "stderr": 6.505, "cost_per_test": 14.876209, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 36.842, "latency": 320.962, "stderr": 6.446, "cost_per_test": 0.345173, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 36.842, "latency": 767.942, "stderr": 6.446, "cost_per_test": 0.321616, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k3": { "accuracy": 36.842, "latency": 772.67, "stderr": 6.446, "cost_per_test": 3.543085, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.5": { "accuracy": 36.842, "latency": 874.891, "stderr": 6.446, "cost_per_test": 0.822469, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 36.842, "latency": 989.26, "stderr": 6.446, "cost_per_test": 1.99149, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 36.842, "latency": 2310.474, "stderr": 6.446, "cost_per_test": 7.322657, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 36.842, "latency": 3744.173, "stderr": 6.446, "cost_per_test": 0.819373, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 35.088, "latency": 540.381, "stderr": 6.377, "cost_per_test": 0.616709, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-spark-open": { "accuracy": 35.088, "latency": 2183.678, "stderr": 6.377, "cost_per_test": 2.568801, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-legal": { "accuracy": 35.088, "latency": 2208.959, "stderr": 6.377, "cost_per_test": 0.046529, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 35.088, "latency": 2224.103, "stderr": 6.377, "cost_per_test": 0.715843, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 35.088, "latency": 2341.734, "stderr": 6.377, "cost_per_test": 3.38256, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 35.088, "latency": 2498.94, "stderr": 6.377, "cost_per_test": 1.017486, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 35.088, "latency": 4543.691, "stderr": 6.377, "cost_per_test": 8.912211, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 33.333, "latency": 347.571, "stderr": 6.299, "cost_per_test": 0.827264, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 33.333, "latency": 682.99, "stderr": 6.299, "cost_per_test": 0.233636, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 33.333, "latency": 1712.774, "stderr": 6.299, "cost_per_test": 1.169956, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 33.333, "latency": 2749.369, "stderr": 6.299, "cost_per_test": 0.250408, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4": { "accuracy": 33.333, "latency": 5104.912, "stderr": 6.299, "cost_per_test": 2.104097, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "fireworks/ember-1": { "accuracy": 31.579, "latency": 2079.561, "stderr": 6.212, "cost_per_test": 3.601082, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 31.579, "latency": 5432.346, "stderr": 6.212, "cost_per_test": 3.124013, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-5.1": { "accuracy": 29.825, "latency": 331.685, "stderr": 6.113, "cost_per_test": 0.405872, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "inception/mercury-3": { "accuracy": 29.825, "latency": 425.103, "stderr": 6.113, "cost_per_test": 0.06767, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 29.825, "latency": 529.681, "stderr": 6.113, "cost_per_test": 2.779258, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 29.825, "latency": 1025.226, "stderr": 6.113, "cost_per_test": 0.071477, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-astra": { "accuracy": 29.825, "latency": 1658.26, "stderr": 6.113, "cost_per_test": 10.941763, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 29.825, "latency": 2563.313, "stderr": 6.113, "cost_per_test": 0.994665, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 28.07, "latency": 6680.114, "stderr": 6.005, "cost_per_test": 8.328752, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 26.316, "latency": 256.075, "stderr": 5.884, "cost_per_test": 1.359179, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 26.316, "latency": 855.631, "stderr": 5.884, "cost_per_test": 0.325282, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 26.316, "latency": 1433.199, "stderr": 5.884, "cost_per_test": 1.995877, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-sol": { "accuracy": 26.316, "latency": 1562.595, "stderr": 5.884, "cost_per_test": 5.495004, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 26.316, "latency": 2463.873, "stderr": 5.884, "cost_per_test": 0.454694, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 26.316, "latency": 2805.101, "stderr": 5.884, "cost_per_test": 0.425544, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 24.561, "latency": 189.608, "stderr": 5.752, "cost_per_test": 0.501214, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 24.561, "latency": 711.836, "stderr": 5.752, "cost_per_test": 0.143249, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 24.561, "latency": 1264.855, "stderr": 5.752, "cost_per_test": 0.25224, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 22.807, "latency": 1034.653, "stderr": 5.607, "cost_per_test": 0.846458, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 21.053, "latency": 787.984, "stderr": 5.448, "cost_per_test": 1.332257, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 21.053, "latency": 872.79, "stderr": 5.448, "cost_per_test": 0.509253, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "reflection/Beam-1.0": { "accuracy": 19.298, "latency": 87.152, "stderr": 5.274, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 19.298, "latency": 217.466, "stderr": 5.274, "cost_per_test": 0.48486, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 17.544, "latency": 267.815, "stderr": 5.083, "cost_per_test": 0.155562, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 17.544, "latency": 4308.911, "stderr": 5.083, "cost_per_test": 3.298261, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 15.789, "latency": 651.134, "stderr": 4.873, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 15.789, "latency": 858.129, "stderr": 4.873, "cost_per_test": 0.2621, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 15.789, "latency": 1004.085, "stderr": 4.873, "cost_per_test": 0.615927, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 15.789, "latency": 1398.791, "stderr": 4.873, "cost_per_test": 0.25486, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.3": { "accuracy": 14.035, "latency": 110.22, "stderr": 4.642, "cost_per_test": 0.199087, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 14.035, "latency": 398.106, "stderr": 4.642, "cost_per_test": 0.081998, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 14.035, "latency": 1937.218, "stderr": 4.642, "cost_per_test": 1.608029, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 12.281, "latency": 63.499, "stderr": 4.386, "cost_per_test": 0.122864, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 12.281, "latency": 221.166, "stderr": 4.386, "cost_per_test": 0.13559, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 12.281, "latency": 435.186, "stderr": 4.386, "cost_per_test": 0.022099, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 10.526, "latency": 283.18, "stderr": 4.101, "cost_per_test": 0.260687, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 10.526, "latency": 611.807, "stderr": 4.101, "cost_per_test": 1.490522, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 10.526, "latency": 698.746, "stderr": 4.101, "cost_per_test": 0.68798, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 8.772, "latency": 78.654, "stderr": 3.78, "cost_per_test": 0.062754, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 7.018, "latency": 394.953, "stderr": 3.413, "cost_per_test": 0.023143, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 5.263, "latency": 421.649, "stderr": 2.984, "cost_per_test": 0.226204, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 5.263, "latency": 2330.292, "stderr": 2.984, "cost_per_test": 2.024329, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 3.509, "latency": 187.227, "stderr": 2.459, "cost_per_test": 0.003839, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 3.509, "latency": 548.936, "stderr": 2.459, "cost_per_test": 0.153371, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 1.754, "latency": 20.102, "stderr": 1.754, "cost_per_test": 0.013186, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 1.754, "latency": 48.522, "stderr": 1.754, "cost_per_test": 0.059734, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 185.804, "stderr": 0.0, "cost_per_test": 0.034171, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 301.868, "stderr": 0.0, "cost_per_test": 0.108798, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 521.807, "stderr": 0.0, "cost_per_test": 0.26163, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } }, "family": { "google/gemini-4-carbon": { "accuracy": 45.455, "latency": 4551.243, "stderr": 10.866, "cost_per_test": 12.648487, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 40.909, "latency": 357.472, "stderr": 10.729, "cost_per_test": 0.633898, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 40.909, "latency": 1858.312, "stderr": 10.729, "cost_per_test": 6.195885, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-legal": { "accuracy": 40.909, "latency": 2827.87, "stderr": 10.729, "cost_per_test": 0.051257, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse-wonder": { "accuracy": 36.364, "latency": 1437.652, "stderr": 10.497, "cost_per_test": 5.323844, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 36.364, "latency": 1643.011, "stderr": 10.497, "cost_per_test": 11.011763, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 36.364, "latency": 2821.52, "stderr": 10.497, "cost_per_test": 10.91868, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 36.364, "latency": 8086.246, "stderr": 10.497, "cost_per_test": 5.806978, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 31.818, "latency": 638.032, "stderr": 10.164, "cost_per_test": 3.584147, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 31.818, "latency": 1329.086, "stderr": 10.164, "cost_per_test": 0.070993, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-wonder-full": { "accuracy": 31.818, "latency": 2250.577, "stderr": 10.164, "cost_per_test": 4.730031, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 31.818, "latency": 3235.665, "stderr": 10.164, "cost_per_test": 21.529237, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 31.818, "latency": 3849.877, "stderr": 10.164, "cost_per_test": 2.697912, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 31.818, "latency": 4670.091, "stderr": 10.164, "cost_per_test": 0.970682, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 27.273, "latency": 328.141, "stderr": 9.719, "cost_per_test": 0.452803, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 27.273, "latency": 716.66, "stderr": 9.719, "cost_per_test": 0.822081, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 27.273, "latency": 1189.264, "stderr": 9.719, "cost_per_test": 2.408033, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 27.273, "latency": 2445.946, "stderr": 9.719, "cost_per_test": 0.826784, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 27.273, "latency": 2649.862, "stderr": 9.719, "cost_per_test": 0.231872, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/ember-1": { "accuracy": 27.273, "latency": 3035.777, "stderr": 9.719, "cost_per_test": 5.439212, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 27.273, "latency": 3202.004, "stderr": 9.719, "cost_per_test": 0.286053, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4": { "accuracy": 27.273, "latency": 6801.677, "stderr": 9.719, "cost_per_test": 2.756433, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 22.727, "latency": 348.059, "stderr": 9.145, "cost_per_test": 0.406411, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 22.727, "latency": 855.894, "stderr": 9.145, "cost_per_test": 0.383913, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k3": { "accuracy": 22.727, "latency": 966.329, "stderr": 9.145, "cost_per_test": 4.336354, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.6": { "accuracy": 22.727, "latency": 1405.167, "stderr": 9.145, "cost_per_test": 1.425494, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 22.727, "latency": 1713.026, "stderr": 9.145, "cost_per_test": 2.549692, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.2": { "accuracy": 22.727, "latency": 1806.747, "stderr": 9.145, "cost_per_test": 1.50639, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 22.727, "latency": 2019.893, "stderr": 9.145, "cost_per_test": 0.418234, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 22.727, "latency": 3456.123, "stderr": 9.145, "cost_per_test": 1.536833, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 22.727, "latency": 5152.106, "stderr": 9.145, "cost_per_test": 1.724456, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 22.727, "latency": 7441.78, "stderr": 9.145, "cost_per_test": 25.835741, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 18.182, "latency": 121.037, "stderr": 8.417, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 18.182, "latency": 306.75, "stderr": 8.417, "cost_per_test": 1.594003, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 18.182, "latency": 613.145, "stderr": 8.417, "cost_per_test": 2.522005, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 18.182, "latency": 787.574, "stderr": 8.417, "cost_per_test": 2.857814, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 18.182, "latency": 1482.157, "stderr": 8.417, "cost_per_test": 5.56694, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 18.182, "latency": 1564.903, "stderr": 8.417, "cost_per_test": 2.758903, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 18.182, "latency": 1762.575, "stderr": 8.417, "cost_per_test": 0.33418, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 18.182, "latency": 1896.183, "stderr": 8.417, "cost_per_test": 15.003346, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 18.182, "latency": 2357.557, "stderr": 8.417, "cost_per_test": 3.772956, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 18.182, "latency": 3017.714, "stderr": 8.417, "cost_per_test": 1.168665, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 18.182, "latency": 4757.56, "stderr": 8.417, "cost_per_test": 3.063697, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 18.182, "latency": 5063.727, "stderr": 8.417, "cost_per_test": 9.812737, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 18.182, "latency": 5320.428, "stderr": 8.417, "cost_per_test": 12.577804, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 13.636, "latency": 390.844, "stderr": 7.489, "cost_per_test": 0.986437, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 13.636, "latency": 951.192, "stderr": 7.489, "cost_per_test": 0.370092, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 13.636, "latency": 1013.398, "stderr": 7.489, "cost_per_test": 0.323772, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "thinkingmachines/inkling": { "accuracy": 13.636, "latency": 1101.097, "stderr": 7.489, "cost_per_test": 0.562977, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 13.636, "latency": 2332.251, "stderr": 7.489, "cost_per_test": 1.856297, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.5": { "accuracy": 13.636, "latency": 2680.585, "stderr": 7.489, "cost_per_test": 7.622096, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 13.636, "latency": 3983.979, "stderr": 7.489, "cost_per_test": 0.672496, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 13.636, "latency": 6569.663, "stderr": 7.489, "cost_per_test": 30.084962, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 13.636, "latency": 6905.045, "stderr": 7.489, "cost_per_test": 8.216145, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 9.091, "latency": 195.123, "stderr": 6.273, "cost_per_test": 0.581541, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 9.091, "latency": 342.126, "stderr": 6.273, "cost_per_test": 0.41232, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "inception/mercury-3": { "accuracy": 9.091, "latency": 514.571, "stderr": 6.273, "cost_per_test": 0.08758, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "grok/grok-4.5": { "accuracy": 9.091, "latency": 839.919, "stderr": 6.273, "cost_per_test": 0.991333, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 9.091, "latency": 900.57, "stderr": 6.273, "cost_per_test": 1.420719, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 9.091, "latency": 1317.912, "stderr": 6.273, "cost_per_test": 0.816394, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 9.091, "latency": 1722.37, "stderr": 6.273, "cost_per_test": 0.30233, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 9.091, "latency": 1758.115, "stderr": 6.273, "cost_per_test": 5.389618, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 9.091, "latency": 2391.966, "stderr": 6.273, "cost_per_test": 0.88974, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-luna": { "accuracy": 9.091, "latency": 2917.375, "stderr": 6.273, "cost_per_test": 0.465031, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 9.091, "latency": 6312.514, "stderr": 6.273, "cost_per_test": 3.697863, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 4.545, "latency": 62.256, "stderr": 4.545, "cost_per_test": 0.169797, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 4.545, "latency": 98.358, "stderr": 4.545, "cost_per_test": 0.074199, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 4.545, "latency": 167.284, "stderr": 4.545, "cost_per_test": 0.168183, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 4.545, "latency": 434.377, "stderr": 4.545, "cost_per_test": 4.138935, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 4.545, "latency": 435.98, "stderr": 4.545, "cost_per_test": 0.075731, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 4.545, "latency": 458.965, "stderr": 4.545, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 4.545, "latency": 692.635, "stderr": 4.545, "cost_per_test": 0.039086, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 4.545, "latency": 1001.225, "stderr": 4.545, "cost_per_test": 0.173654, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 4.545, "latency": 4690.594, "stderr": 4.545, "cost_per_test": 3.506565, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 18.535, "stderr": 0.0, "cost_per_test": 0.013412, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 47.19, "stderr": 0.0, "cost_per_test": 0.062993, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 86.589, "stderr": 0.0, "cost_per_test": 0.230167, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 222.566, "stderr": 0.0, "cost_per_test": 0.036939, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 246.131, "stderr": 0.0, "cost_per_test": 0.16395, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 250.897, "stderr": 0.0, "cost_per_test": 0.288946, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 311.615, "stderr": 0.0, "cost_per_test": 0.005312, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 383.551, "stderr": 0.0, "cost_per_test": 0.13236, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 422.665, "stderr": 0.0, "cost_per_test": 0.030982, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 436.535, "stderr": 0.0, "cost_per_test": 0.255715, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 532.251, "stderr": 0.0, "cost_per_test": 0.261764, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 570.598, "stderr": 0.0, "cost_per_test": 1.090579, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 592.178, "stderr": 0.0, "cost_per_test": 0.140683, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 727.744, "stderr": 0.0, "cost_per_test": 1.001303, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 1055.917, "stderr": 0.0, "cost_per_test": 0.443026, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 2863.795, "stderr": 0.0, "cost_per_test": 2.335677, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 4205.69, "stderr": 0.0, "cost_per_test": 2.933958, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null } }, "health": { "fireworks/ember-1": { "accuracy": 80.0, "latency": 2538.105, "stderr": 7.428, "cost_per_test": 4.232271, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 80.0, "latency": 3030.813, "stderr": 7.428, "cost_per_test": 22.907678, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 80.0, "latency": 3850.231, "stderr": 7.428, "cost_per_test": 8.132992, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 80.0, "latency": 4151.253, "stderr": 7.428, "cost_per_test": 22.836198, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 80.0, "latency": 4630.853, "stderr": 7.428, "cost_per_test": 12.659413, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 80.0, "latency": 5675.961, "stderr": 7.428, "cost_per_test": 20.596615, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 76.667, "latency": 836.886, "stderr": 7.854, "cost_per_test": 4.339603, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse-wonder": { "accuracy": 76.667, "latency": 858.216, "stderr": 7.854, "cost_per_test": 4.579894, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-astra": { "accuracy": 76.667, "latency": 986.856, "stderr": 7.854, "cost_per_test": 10.254175, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 76.667, "latency": 1095.281, "stderr": 7.854, "cost_per_test": 4.723349, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 76.667, "latency": 1338.036, "stderr": 7.854, "cost_per_test": 4.625213, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 76.667, "latency": 1346.948, "stderr": 7.854, "cost_per_test": 7.009258, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 76.667, "latency": 1414.958, "stderr": 7.854, "cost_per_test": 3.858235, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 76.667, "latency": 3215.749, "stderr": 7.854, "cost_per_test": 2.470231, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 73.333, "latency": 318.119, "stderr": 8.212, "cost_per_test": 0.64171, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 73.333, "latency": 706.355, "stderr": 8.212, "cost_per_test": 0.272236, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.6": { "accuracy": 73.333, "latency": 1693.184, "stderr": 8.212, "cost_per_test": 1.828426, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 73.333, "latency": 3278.164, "stderr": 8.212, "cost_per_test": 0.238934, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 73.333, "latency": 3496.535, "stderr": 8.212, "cost_per_test": 2.5406, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 73.333, "latency": 5711.873, "stderr": 8.212, "cost_per_test": 6.862578, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 70.0, "latency": 470.162, "stderr": 8.51, "cost_per_test": 0.708927, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 70.0, "latency": 710.57, "stderr": 8.51, "cost_per_test": 0.327988, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 70.0, "latency": 1360.302, "stderr": 8.51, "cost_per_test": 0.193071, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 70.0, "latency": 2414.0, "stderr": 8.51, "cost_per_test": 0.809076, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-carbon": { "accuracy": 70.0, "latency": 2450.538, "stderr": 8.51, "cost_per_test": 6.751201, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 66.667, "latency": 1126.576, "stderr": 8.754, "cost_per_test": 12.38182, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 66.667, "latency": 1266.05, "stderr": 8.754, "cost_per_test": 2.204201, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 66.667, "latency": 1553.826, "stderr": 8.754, "cost_per_test": 0.926438, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 66.667, "latency": 1838.285, "stderr": 8.754, "cost_per_test": 3.207198, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 66.667, "latency": 1923.984, "stderr": 8.754, "cost_per_test": 0.708547, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 66.667, "latency": 2062.802, "stderr": 8.754, "cost_per_test": 9.360291, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 66.667, "latency": 3508.124, "stderr": 8.754, "cost_per_test": 1.530774, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 66.667, "latency": 3604.391, "stderr": 8.754, "cost_per_test": 0.88574, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 63.333, "latency": 1364.3, "stderr": 8.949, "cost_per_test": 2.657382, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 63.333, "latency": 1922.826, "stderr": 8.949, "cost_per_test": 2.642852, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "tencent/hy4": { "accuracy": 63.333, "latency": 5571.399, "stderr": 8.949, "cost_per_test": 2.397715, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-sol": { "accuracy": 60.0, "latency": 1026.417, "stderr": 9.097, "cost_per_test": 3.793295, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-legal": { "accuracy": 60.0, "latency": 2681.332, "stderr": 9.097, "cost_per_test": 0.057691, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 60.0, "latency": 2705.795, "stderr": 9.097, "cost_per_test": 1.314815, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 56.667, "latency": 282.896, "stderr": 9.202, "cost_per_test": 1.899416, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 56.667, "latency": 303.781, "stderr": 9.202, "cost_per_test": 0.574895, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 56.667, "latency": 3359.798, "stderr": 9.202, "cost_per_test": 3.11111, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 56.667, "latency": 7418.628, "stderr": 9.202, "cost_per_test": 5.193942, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 53.333, "latency": 462.415, "stderr": 9.264, "cost_per_test": 4.040025, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 53.333, "latency": 663.425, "stderr": 9.264, "cost_per_test": 3.159294, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 53.333, "latency": 781.127, "stderr": 9.264, "cost_per_test": 0.081431, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 53.333, "latency": 1047.406, "stderr": 9.264, "cost_per_test": 3.45972, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 53.333, "latency": 1685.594, "stderr": 9.264, "cost_per_test": 1.561613, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 53.333, "latency": 2028.962, "stderr": 9.264, "cost_per_test": 0.905473, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-luna": { "accuracy": 53.333, "latency": 2156.169, "stderr": 9.264, "cost_per_test": 0.374147, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 50.0, "latency": 245.802, "stderr": 9.285, "cost_per_test": 0.701623, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.2": { "accuracy": 50.0, "latency": 931.988, "stderr": 9.285, "cost_per_test": 0.931196, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 50.0, "latency": 5897.488, "stderr": 9.285, "cost_per_test": 2.500509, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 46.667, "latency": 97.234, "stderr": 9.264, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "inception/mercury-3": { "accuracy": 46.667, "latency": 472.047, "stderr": 9.264, "cost_per_test": 0.076792, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 46.667, "latency": 800.16, "stderr": 9.264, "cost_per_test": 0.394232, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 46.667, "latency": 999.255, "stderr": 9.264, "cost_per_test": 0.81017, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 43.333, "latency": 274.578, "stderr": 9.202, "cost_per_test": 1.402227, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling": { "accuracy": 43.333, "latency": 910.936, "stderr": 9.202, "cost_per_test": 0.496174, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.3": { "accuracy": 40.0, "latency": 79.862, "stderr": 9.097, "cost_per_test": 0.233416, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 40.0, "latency": 298.421, "stderr": 9.097, "cost_per_test": 0.506452, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 40.0, "latency": 848.765, "stderr": 9.097, "cost_per_test": 1.786099, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 36.667, "latency": 134.685, "stderr": 8.949, "cost_per_test": 0.416767, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 36.667, "latency": 322.342, "stderr": 8.949, "cost_per_test": 0.321248, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 36.667, "latency": 1384.623, "stderr": 8.949, "cost_per_test": 0.400672, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 36.667, "latency": 3297.206, "stderr": 8.949, "cost_per_test": 0.553927, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 33.333, "latency": 232.285, "stderr": 8.754, "cost_per_test": 1.804209, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 30.0, "latency": 145.057, "stderr": 8.51, "cost_per_test": 0.136684, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 30.0, "latency": 696.803, "stderr": 8.51, "cost_per_test": 0.848896, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 30.0, "latency": 998.171, "stderr": 8.51, "cost_per_test": 0.155212, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 30.0, "latency": 1608.673, "stderr": 8.51, "cost_per_test": 1.371176, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 26.667, "latency": 477.359, "stderr": 8.212, "cost_per_test": 0.028475, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 26.667, "latency": 830.651, "stderr": 8.212, "cost_per_test": 0.505144, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 23.333, "latency": 387.77, "stderr": 7.854, "cost_per_test": 0.119406, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 16.667, "latency": 538.653, "stderr": 6.92, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 16.667, "latency": 1671.994, "stderr": 6.92, "cost_per_test": 1.329631, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 13.333, "latency": 48.195, "stderr": 6.312, "cost_per_test": 0.13408, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 13.333, "latency": 334.244, "stderr": 6.312, "cost_per_test": 0.106495, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 13.333, "latency": 587.499, "stderr": 6.312, "cost_per_test": 0.21973, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 10.0, "latency": 71.497, "stderr": 5.571, "cost_per_test": 0.055137, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 10.0, "latency": 229.682, "stderr": 5.571, "cost_per_test": 0.532775, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 10.0, "latency": 1152.919, "stderr": 5.571, "cost_per_test": 0.18555, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 6.667, "latency": 19.508, "stderr": 4.632, "cost_per_test": 0.013992, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 6.667, "latency": 188.923, "stderr": 4.632, "cost_per_test": 0.290063, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 6.667, "latency": 372.619, "stderr": 4.632, "cost_per_test": 0.056492, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 6.667, "latency": 618.768, "stderr": 4.632, "cost_per_test": 1.063116, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 3.333, "latency": 51.629, "stderr": 3.333, "cost_per_test": 0.048442, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-m.1": { "accuracy": 3.333, "latency": 474.816, "stderr": 3.333, "cost_per_test": 0.199185, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 232.734, "stderr": 0.0, "cost_per_test": 0.036118, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 396.297, "stderr": 0.0, "cost_per_test": 0.09495, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 661.96, "stderr": 0.0, "cost_per_test": 0.005274, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "immigration": { "meta/muse_spark_1_2": { "accuracy": 54.545, "latency": 372.58, "stderr": 15.746, "cost_per_test": 1.011872, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 54.545, "latency": 1545.533, "stderr": 15.746, "cost_per_test": 4.782473, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 54.545, "latency": 2227.72, "stderr": 15.746, "cost_per_test": 2.139853, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 54.545, "latency": 7783.489, "stderr": 15.746, "cost_per_test": 28.846858, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 45.455, "latency": 377.453, "stderr": 15.746, "cost_per_test": 0.749015, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 45.455, "latency": 852.202, "stderr": 15.746, "cost_per_test": 4.505049, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse-wonder": { "accuracy": 45.455, "latency": 1055.89, "stderr": 15.746, "cost_per_test": 4.615585, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 45.455, "latency": 1309.204, "stderr": 15.746, "cost_per_test": 5.041009, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-4-carbon": { "accuracy": 45.455, "latency": 2577.035, "stderr": 15.746, "cost_per_test": 7.345528, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 45.455, "latency": 2999.169, "stderr": 15.746, "cost_per_test": 1.324131, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 45.455, "latency": 3576.368, "stderr": 15.746, "cost_per_test": 0.63176, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 45.455, "latency": 3943.62, "stderr": 15.746, "cost_per_test": 3.006031, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 45.455, "latency": 4168.205, "stderr": 15.746, "cost_per_test": 0.281372, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 45.455, "latency": 5565.396, "stderr": 15.746, "cost_per_test": 29.782421, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 45.455, "latency": 6121.239, "stderr": 15.746, "cost_per_test": 15.318633, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 36.364, "latency": 145.322, "stderr": 15.212, "cost_per_test": 0.576237, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 36.364, "latency": 522.726, "stderr": 15.212, "cost_per_test": 3.815712, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 36.364, "latency": 771.306, "stderr": 15.212, "cost_per_test": 1.548677, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 36.364, "latency": 1059.33, "stderr": 15.212, "cost_per_test": 0.440884, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "thinkingmachines/inkling": { "accuracy": 36.364, "latency": 1086.925, "stderr": 15.212, "cost_per_test": 0.562099, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 36.364, "latency": 1290.179, "stderr": 15.212, "cost_per_test": 9.559148, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 36.364, "latency": 1629.748, "stderr": 15.212, "cost_per_test": 2.754829, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 36.364, "latency": 1914.142, "stderr": 15.212, "cost_per_test": 6.6645, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 36.364, "latency": 2079.496, "stderr": 15.212, "cost_per_test": 14.716249, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-legal": { "accuracy": 36.364, "latency": 2188.906, "stderr": 15.212, "cost_per_test": 0.050144, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 36.364, "latency": 2309.031, "stderr": 15.212, "cost_per_test": 5.522954, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 36.364, "latency": 2363.885, "stderr": 15.212, "cost_per_test": 0.899309, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 36.364, "latency": 2512.615, "stderr": 15.212, "cost_per_test": 8.597148, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 36.364, "latency": 2795.76, "stderr": 15.212, "cost_per_test": 1.125625, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 36.364, "latency": 4229.709, "stderr": 15.212, "cost_per_test": 29.748136, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 36.364, "latency": 5391.649, "stderr": 15.212, "cost_per_test": 2.04155, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 27.273, "latency": 128.854, "stderr": 14.084, "cost_per_test": 0.088481, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 27.273, "latency": 176.78, "stderr": 14.084, "cost_per_test": 0.283317, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 27.273, "latency": 231.193, "stderr": 14.084, "cost_per_test": 1.096453, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 27.273, "latency": 376.011, "stderr": 14.084, "cost_per_test": 0.081151, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 27.273, "latency": 586.852, "stderr": 14.084, "cost_per_test": 0.921916, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 27.273, "latency": 659.266, "stderr": 14.084, "cost_per_test": 0.026153, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 27.273, "latency": 728.3, "stderr": 14.084, "cost_per_test": 3.441413, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 27.273, "latency": 733.81, "stderr": 14.084, "cost_per_test": 0.307991, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 27.273, "latency": 866.165, "stderr": 14.084, "cost_per_test": 0.178499, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 27.273, "latency": 882.27, "stderr": 14.084, "cost_per_test": 0.482243, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 27.273, "latency": 1048.83, "stderr": 14.084, "cost_per_test": 2.703862, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 27.273, "latency": 1192.0, "stderr": 14.084, "cost_per_test": 1.003117, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 27.273, "latency": 1260.914, "stderr": 14.084, "cost_per_test": 0.296663, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-spark-open": { "accuracy": 27.273, "latency": 1537.666, "stderr": 14.084, "cost_per_test": 0.570169, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 27.273, "latency": 1793.408, "stderr": 14.084, "cost_per_test": 0.234002, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 27.273, "latency": 2067.692, "stderr": 14.084, "cost_per_test": 0.952254, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 27.273, "latency": 2091.212, "stderr": 14.084, "cost_per_test": 8.705325, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 27.273, "latency": 2203.778, "stderr": 14.084, "cost_per_test": 2.644131, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 27.273, "latency": 4515.312, "stderr": 14.084, "cost_per_test": 1.089, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3": { "accuracy": 27.273, "latency": 4767.048, "stderr": 14.084, "cost_per_test": 3.223316, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 27.273, "latency": 5257.651, "stderr": 14.084, "cost_per_test": 3.326554, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 27.273, "latency": 7303.148, "stderr": 14.084, "cost_per_test": 4.997493, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 18.182, "latency": 60.521, "stderr": 12.197, "cost_per_test": 0.102995, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 18.182, "latency": 69.703, "stderr": 12.197, "cost_per_test": 0.06269, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "reflection/Beam-1.0": { "accuracy": 18.182, "latency": 71.817, "stderr": 12.197, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 18.182, "latency": 147.165, "stderr": 12.197, "cost_per_test": 0.405548, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 18.182, "latency": 185.396, "stderr": 12.197, "cost_per_test": 0.389491, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 18.182, "latency": 219.971, "stderr": 12.197, "cost_per_test": 0.884226, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 18.182, "latency": 255.303, "stderr": 12.197, "cost_per_test": 0.300956, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.1": { "accuracy": 18.182, "latency": 389.141, "stderr": 12.197, "cost_per_test": 0.560419, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "inception/mercury-3": { "accuracy": 18.182, "latency": 449.603, "stderr": 12.197, "cost_per_test": 0.076601, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 18.182, "latency": 598.99, "stderr": 12.197, "cost_per_test": 0.650754, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 18.182, "latency": 705.891, "stderr": 12.197, "cost_per_test": 1.495632, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 18.182, "latency": 812.146, "stderr": 12.197, "cost_per_test": 0.150825, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 18.182, "latency": 823.783, "stderr": 12.197, "cost_per_test": 0.603054, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 18.182, "latency": 1272.83, "stderr": 12.197, "cost_per_test": 0.124704, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 18.182, "latency": 1561.245, "stderr": 12.197, "cost_per_test": 1.49714, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 18.182, "latency": 2027.42, "stderr": 12.197, "cost_per_test": 1.631269, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 18.182, "latency": 2445.483, "stderr": 12.197, "cost_per_test": 3.066554, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 18.182, "latency": 2966.232, "stderr": 12.197, "cost_per_test": 1.017586, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 18.182, "latency": 3025.432, "stderr": 12.197, "cost_per_test": 5.242893, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 18.182, "latency": 3289.995, "stderr": 12.197, "cost_per_test": 0.523583, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 18.182, "latency": 5146.748, "stderr": 12.197, "cost_per_test": 9.402071, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4": { "accuracy": 18.182, "latency": 6986.35, "stderr": 12.197, "cost_per_test": 2.97227, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 18.182, "latency": 7116.076, "stderr": 12.197, "cost_per_test": 8.488087, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 9.091, "latency": 21.669, "stderr": 9.091, "cost_per_test": 0.014207, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 9.091, "latency": 242.659, "stderr": 9.091, "cost_per_test": 0.366775, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 9.091, "latency": 305.943, "stderr": 9.091, "cost_per_test": 0.448482, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 9.091, "latency": 473.789, "stderr": 9.091, "cost_per_test": 0.036168, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 9.091, "latency": 848.642, "stderr": 9.091, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 9.091, "latency": 851.042, "stderr": 9.091, "cost_per_test": 1.91176, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 9.091, "latency": 902.658, "stderr": 9.091, "cost_per_test": 0.700549, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 9.091, "latency": 5272.469, "stderr": 9.091, "cost_per_test": 3.88229, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 38.568, "stderr": 0.0, "cost_per_test": 0.046126, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 173.024, "stderr": 0.0, "cost_per_test": 0.003083, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 201.446, "stderr": 0.0, "cost_per_test": 0.033288, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 356.501, "stderr": 0.0, "cost_per_test": 0.229174, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 394.752, "stderr": 0.0, "cost_per_test": 0.099916, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 559.812, "stderr": 0.0, "cost_per_test": 0.159001, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 610.78, "stderr": 0.0, "cost_per_test": 0.296042, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } }, "weighted_pass": { "google/gemini-4-carbon": { "accuracy": 91.609, "latency": 3112.579, "stderr": 1.026, "cost_per_test": 8.743524, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-4-argon": { "accuracy": 90.991, "latency": 1881.045, "stderr": 1.028, "cost_per_test": 6.44908, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 90.577, "latency": 1860.719, "stderr": 1.151, "cost_per_test": 6.762498, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 90.213, "latency": 3449.382, "stderr": 1.224, "cost_per_test": 23.055789, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 89.804, "latency": 1654.378, "stderr": 1.212, "cost_per_test": 3.595401, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 88.346, "latency": 1504.077, "stderr": 1.25, "cost_per_test": 1.530204, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 87.872, "latency": 735.191, "stderr": 1.32, "cost_per_test": 0.288586, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 87.816, "latency": 4631.383, "stderr": 1.262, "cost_per_test": 21.605566, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 87.765, "latency": 1337.604, "stderr": 1.32, "cost_per_test": 9.791426, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 87.732, "latency": 3110.171, "stderr": 1.376, "cost_per_test": 2.241388, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse-wonder": { "accuracy": 87.726, "latency": 927.111, "stderr": 1.368, "cost_per_test": 3.980242, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 87.541, "latency": 3067.486, "stderr": 1.273, "cost_per_test": 0.240392, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 87.457, "latency": 324.318, "stderr": 1.293, "cost_per_test": 1.630498, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 86.8, "latency": 317.461, "stderr": 1.569, "cost_per_test": 0.588812, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 86.551, "latency": 1274.337, "stderr": 1.374, "cost_per_test": 4.920812, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 86.28, "latency": 2106.881, "stderr": 1.376, "cost_per_test": 0.69856, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-legal": { "accuracy": 86.146, "latency": 2388.741, "stderr": 1.44, "cost_per_test": 0.04878, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 86.087, "latency": 3677.636, "stderr": 1.463, "cost_per_test": 2.486351, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 86.067, "latency": 714.036, "stderr": 1.441, "cost_per_test": 2.820942, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 86.027, "latency": 2313.552, "stderr": 1.32, "cost_per_test": 3.674091, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 85.989, "latency": 1763.023, "stderr": 1.394, "cost_per_test": 1.559385, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 85.493, "latency": 1816.322, "stderr": 1.537, "cost_per_test": 0.181201, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 85.339, "latency": 989.604, "stderr": 1.359, "cost_per_test": 2.302948, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 85.271, "latency": 5571.099, "stderr": 1.387, "cost_per_test": 2.723732, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 85.266, "latency": 306.893, "stderr": 1.523, "cost_per_test": 0.508097, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 85.209, "latency": 530.813, "stderr": 1.592, "cost_per_test": 0.650063, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "tencent/hy4-preview": { "accuracy": 85.176, "latency": 3839.511, "stderr": 1.486, "cost_per_test": 0.846372, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 85.092, "latency": 4172.869, "stderr": 1.359, "cost_per_test": 7.907299, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 85.047, "latency": 2373.859, "stderr": 1.285, "cost_per_test": 0.845008, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 84.765, "latency": 1491.339, "stderr": 1.293, "cost_per_test": 10.577143, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 84.764, "latency": 4979.806, "stderr": 1.505, "cost_per_test": 1.647359, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 84.759, "latency": 2168.413, "stderr": 1.471, "cost_per_test": 0.838645, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "kimi/kimi-k3": { "accuracy": 84.643, "latency": 760.122, "stderr": 1.483, "cost_per_test": 3.469601, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 84.621, "latency": 312.994, "stderr": 1.419, "cost_per_test": 0.377379, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 84.592, "latency": 1545.997, "stderr": 1.431, "cost_per_test": 2.719446, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 84.522, "latency": 1704.14, "stderr": 1.421, "cost_per_test": 1.09052, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 84.503, "latency": 701.783, "stderr": 1.427, "cost_per_test": 0.24793, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.5": { "accuracy": 84.423, "latency": 953.767, "stderr": 1.429, "cost_per_test": 1.086814, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 84.312, "latency": 2060.48, "stderr": 1.321, "cost_per_test": 3.384561, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 84.274, "latency": 5588.989, "stderr": 1.74, "cost_per_test": 13.663179, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 84.132, "latency": 288.96, "stderr": 1.519, "cost_per_test": 0.760397, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 84.089, "latency": 511.955, "stderr": 1.475, "cost_per_test": 3.05609, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 83.964, "latency": 2265.105, "stderr": 1.474, "cost_per_test": 7.402471, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 83.93, "latency": 277.753, "stderr": 1.373, "cost_per_test": 1.305281, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 83.843, "latency": 2527.177, "stderr": 1.518, "cost_per_test": 1.088017, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 83.076, "latency": 2911.983, "stderr": 1.532, "cost_per_test": 0.444777, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "zai/glm-5.2": { "accuracy": 82.871, "latency": 1023.468, "stderr": 1.478, "cost_per_test": 0.889209, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 82.633, "latency": 1600.721, "stderr": 1.581, "cost_per_test": 2.159719, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "tencent/hy4": { "accuracy": 82.605, "latency": 5455.029, "stderr": 1.912, "cost_per_test": 2.222804, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 81.223, "latency": 1266.744, "stderr": 1.513, "cost_per_test": 0.275172, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 81.146, "latency": 813.838, "stderr": 1.532, "cost_per_test": 0.342955, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 80.689, "latency": 1024.619, "stderr": 1.537, "cost_per_test": 0.682218, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 80.669, "latency": 7649.584, "stderr": 2.232, "cost_per_test": 5.312295, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 80.543, "latency": 168.482, "stderr": 1.573, "cost_per_test": 0.466741, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 80.378, "latency": 6749.531, "stderr": 2.238, "cost_per_test": 23.613691, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 80.193, "latency": 869.377, "stderr": 1.522, "cost_per_test": 0.494044, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-3": { "accuracy": 80.072, "latency": 446.517, "stderr": 1.599, "cost_per_test": 0.074351, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 79.577, "latency": 919.689, "stderr": 1.844, "cost_per_test": 0.070424, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "reflection/Beam-1.0": { "accuracy": 79.467, "latency": 94.568, "stderr": 1.578, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "openai/gpt-6-luna": { "accuracy": 79.296, "latency": 2547.747, "stderr": 1.631, "cost_per_test": 0.440296, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 79.181, "latency": 1449.451, "stderr": 1.564, "cost_per_test": 4.897717, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 78.924, "latency": 774.823, "stderr": 1.605, "cost_per_test": 0.141548, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 78.708, "latency": 4202.728, "stderr": 1.61, "cost_per_test": 3.220614, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 78.4, "latency": 808.802, "stderr": 1.689, "cost_per_test": 1.400701, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.1": { "accuracy": 77.808, "latency": 325.824, "stderr": 1.696, "cost_per_test": 0.385737, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 76.61, "latency": 242.05, "stderr": 1.602, "cost_per_test": 1.100145, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 76.0, "latency": 416.437, "stderr": 1.753, "cost_per_test": 0.082648, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 75.174, "latency": 187.7, "stderr": 1.651, "cost_per_test": 0.135395, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 74.98, "latency": 831.995, "stderr": 1.746, "cost_per_test": 0.295949, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 74.742, "latency": 1853.133, "stderr": 1.698, "cost_per_test": 1.520305, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 74.341, "latency": 658.954, "stderr": 1.708, "cost_per_test": 0.697997, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 73.21, "latency": 515.464, "stderr": 1.797, "cost_per_test": 0.028227, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 71.447, "latency": 6237.394, "stderr": 2.566, "cost_per_test": 7.231463, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 71.336, "latency": 1332.514, "stderr": 1.765, "cost_per_test": 0.23035, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 70.148, "latency": 82.971, "stderr": 1.855, "cost_per_test": 0.063761, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "grok/grok-4.3": { "accuracy": 69.491, "latency": 84.583, "stderr": 1.827, "cost_per_test": 0.203143, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 68.939, "latency": 59.197, "stderr": 1.846, "cost_per_test": 0.124723, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 68.216, "latency": 391.469, "stderr": 1.898, "cost_per_test": 0.033318, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 68.005, "latency": 611.366, "stderr": 1.98, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 67.326, "latency": 2256.339, "stderr": 1.837, "cost_per_test": 1.864673, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 65.11, "latency": 244.487, "stderr": 1.904, "cost_per_test": 0.261499, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 64.939, "latency": 243.38, "stderr": 1.946, "cost_per_test": 0.26891, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 64.627, "latency": 211.833, "stderr": 1.989, "cost_per_test": 0.036023, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 63.885, "latency": 420.647, "stderr": 1.987, "cost_per_test": 0.219167, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 63.308, "latency": 613.248, "stderr": 1.96, "cost_per_test": 1.308079, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 62.338, "latency": 466.26, "stderr": 1.84, "cost_per_test": 0.219839, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 61.958, "latency": 526.838, "stderr": 1.95, "cost_per_test": 0.141253, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 58.67, "latency": 47.609, "stderr": 1.894, "cost_per_test": 0.058112, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 56.27, "latency": 352.036, "stderr": 1.989, "cost_per_test": 0.107892, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 54.415, "latency": 20.403, "stderr": 1.956, "cost_per_test": 0.013607, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 48.972, "latency": 285.48, "stderr": 1.864, "cost_per_test": 0.004365, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "weighted_pass__administrative_regulatory": { "google/gemini-4-argon": { "accuracy": 92.945, "latency": 1486.33, "stderr": 1.862, "cost_per_test": 4.807402, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 92.809, "latency": 1691.752, "stderr": 1.778, "cost_per_test": 3.18787, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 92.676, "latency": 1532.62, "stderr": 1.919, "cost_per_test": 6.583578, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 92.674, "latency": 631.256, "stderr": 1.865, "cost_per_test": 0.243078, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-4-carbon": { "accuracy": 92.52, "latency": 2535.332, "stderr": 2.03, "cost_per_test": 6.803146, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 92.519, "latency": 3106.859, "stderr": 2.145, "cost_per_test": 20.305794, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 91.39, "latency": 1372.59, "stderr": 2.119, "cost_per_test": 1.517644, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 91.346, "latency": 4690.9, "stderr": 2.288, "cost_per_test": 11.667322, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 91.277, "latency": 1931.905, "stderr": 1.979, "cost_per_test": 0.652538, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 91.07, "latency": 2956.379, "stderr": 2.219, "cost_per_test": 2.133253, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/ember-1": { "accuracy": 90.94, "latency": 2038.532, "stderr": 2.085, "cost_per_test": 3.174554, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 90.822, "latency": 2848.785, "stderr": 1.911, "cost_per_test": 0.198792, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 90.69, "latency": 4138.255, "stderr": 2.118, "cost_per_test": 19.689902, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder": { "accuracy": 90.571, "latency": 889.246, "stderr": 2.422, "cost_per_test": 3.922674, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 90.562, "latency": 1523.131, "stderr": 2.0, "cost_per_test": 0.921373, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 90.519, "latency": 3192.985, "stderr": 2.201, "cost_per_test": 2.201925, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 90.224, "latency": 1179.378, "stderr": 2.232, "cost_per_test": 9.381311, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-legal": { "accuracy": 90.108, "latency": 2358.521, "stderr": 1.981, "cost_per_test": 0.049642, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 89.252, "latency": 749.583, "stderr": 2.267, "cost_per_test": 3.444134, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 89.06, "latency": 656.731, "stderr": 2.16, "cost_per_test": 0.246492, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 88.759, "latency": 3607.895, "stderr": 2.29, "cost_per_test": 6.866314, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 88.75, "latency": 1001.46, "stderr": 2.226, "cost_per_test": 1.36196, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 88.676, "latency": 3941.13, "stderr": 2.238, "cost_per_test": 1.441703, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 88.497, "latency": 1643.295, "stderr": 2.31, "cost_per_test": 2.2119, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 88.492, "latency": 5582.268, "stderr": 2.189, "cost_per_test": 2.616141, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 88.325, "latency": 3713.178, "stderr": 2.506, "cost_per_test": 0.834838, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 88.147, "latency": 302.672, "stderr": 2.538, "cost_per_test": 0.506511, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-spark-open": { "accuracy": 87.942, "latency": 1671.25, "stderr": 2.515, "cost_per_test": 1.423111, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 87.81, "latency": 1285.722, "stderr": 2.292, "cost_per_test": 2.507811, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 87.76, "latency": 2171.348, "stderr": 2.3, "cost_per_test": 0.862352, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 87.613, "latency": 296.369, "stderr": 2.448, "cost_per_test": 1.661721, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 87.513, "latency": 2245.491, "stderr": 2.206, "cost_per_test": 0.755621, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 87.482, "latency": 1103.075, "stderr": 2.459, "cost_per_test": 8.442858, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 87.402, "latency": 1609.837, "stderr": 2.774, "cost_per_test": 0.17954, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 87.365, "latency": 636.543, "stderr": 2.257, "cost_per_test": 2.731524, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 87.214, "latency": 1016.365, "stderr": 2.694, "cost_per_test": 4.175009, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 87.126, "latency": 285.017, "stderr": 2.45, "cost_per_test": 1.356228, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 87.035, "latency": 7602.624, "stderr": 3.249, "cost_per_test": 5.772668, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 87.011, "latency": 425.156, "stderr": 2.677, "cost_per_test": 3.06182, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 86.797, "latency": 901.446, "stderr": 2.273, "cost_per_test": 2.410326, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 86.761, "latency": 306.876, "stderr": 3.007, "cost_per_test": 0.540035, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.5": { "accuracy": 86.669, "latency": 2093.383, "stderr": 2.717, "cost_per_test": 7.953883, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 86.33, "latency": 2424.284, "stderr": 2.663, "cost_per_test": 1.073701, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 86.153, "latency": 755.036, "stderr": 2.391, "cost_per_test": 0.352926, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5.2": { "accuracy": 85.95, "latency": 836.031, "stderr": 2.455, "cost_per_test": 0.808378, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 85.884, "latency": 481.922, "stderr": 3.022, "cost_per_test": 0.624599, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 85.547, "latency": 1439.965, "stderr": 2.599, "cost_per_test": 2.756725, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 85.427, "latency": 5979.926, "stderr": 3.718, "cost_per_test": 20.862676, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 85.147, "latency": 244.664, "stderr": 2.799, "cost_per_test": 0.704178, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 85.049, "latency": 2966.067, "stderr": 2.723, "cost_per_test": 0.472211, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 84.8, "latency": 3381.142, "stderr": 2.625, "cost_per_test": 2.810214, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 84.144, "latency": 877.92, "stderr": 2.374, "cost_per_test": 0.44143, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 83.971, "latency": 789.443, "stderr": 2.671, "cost_per_test": 1.359392, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 83.878, "latency": 1185.766, "stderr": 2.437, "cost_per_test": 0.295871, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 83.623, "latency": 282.392, "stderr": 2.846, "cost_per_test": 0.382532, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 83.517, "latency": 139.396, "stderr": 2.559, "cost_per_test": 0.409841, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 83.372, "latency": 903.624, "stderr": 2.545, "cost_per_test": 0.671217, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 83.349, "latency": 715.742, "stderr": 3.224, "cost_per_test": 0.066967, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 83.175, "latency": 2341.023, "stderr": 2.64, "cost_per_test": 0.409481, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 83.099, "latency": 94.03, "stderr": 2.713, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 82.955, "latency": 842.128, "stderr": 2.398, "cost_per_test": 0.134944, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "tencent/hy4": { "accuracy": 82.011, "latency": 5301.951, "stderr": 4.0, "cost_per_test": 2.211937, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 81.914, "latency": 5673.421, "stderr": 3.783, "cost_per_test": 6.338985, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "inception/mercury-3": { "accuracy": 81.478, "latency": 434.539, "stderr": 2.787, "cost_per_test": 0.075638, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 81.313, "latency": 166.672, "stderr": 2.681, "cost_per_test": 0.129186, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 80.457, "latency": 1112.655, "stderr": 2.933, "cost_per_test": 3.887103, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 80.005, "latency": 307.232, "stderr": 3.081, "cost_per_test": 0.323431, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 79.948, "latency": 805.914, "stderr": 2.796, "cost_per_test": 0.359357, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 79.215, "latency": 219.629, "stderr": 2.633, "cost_per_test": 1.225685, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 77.055, "latency": 381.12, "stderr": 3.191, "cost_per_test": 0.087857, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 76.623, "latency": 625.818, "stderr": 2.914, "cost_per_test": 0.670062, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 75.531, "latency": 1689.904, "stderr": 3.082, "cost_per_test": 1.488125, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 75.15, "latency": 494.675, "stderr": 3.183, "cost_per_test": 0.027887, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.3": { "accuracy": 73.866, "latency": 72.945, "stderr": 3.085, "cost_per_test": 0.19233, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 72.761, "latency": 1153.879, "stderr": 3.058, "cost_per_test": 0.195218, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 72.398, "latency": 476.589, "stderr": 2.881, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 71.96, "latency": 50.778, "stderr": 3.222, "cost_per_test": 0.123387, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 71.929, "latency": 343.766, "stderr": 3.121, "cost_per_test": 0.038807, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 70.856, "latency": 69.443, "stderr": 3.094, "cost_per_test": 0.05243, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 69.72, "latency": 1961.008, "stderr": 3.312, "cost_per_test": 1.456649, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 67.862, "latency": 199.066, "stderr": 3.241, "cost_per_test": 0.274657, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 67.477, "latency": 382.835, "stderr": 3.415, "cost_per_test": 0.200329, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 67.472, "latency": 575.292, "stderr": 3.109, "cost_per_test": 1.130658, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 67.15, "latency": 215.774, "stderr": 3.505, "cost_per_test": 0.032184, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "poolside/laguna-m.1": { "accuracy": 66.556, "latency": 383.823, "stderr": 2.803, "cost_per_test": 0.163947, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 65.678, "latency": 402.197, "stderr": 3.408, "cost_per_test": 0.125736, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 65.064, "latency": 226.406, "stderr": 3.195, "cost_per_test": 0.379263, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 58.074, "latency": 332.137, "stderr": 3.205, "cost_per_test": 0.087793, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "inception/mercury-2.5": { "accuracy": 55.228, "latency": 41.907, "stderr": 3.324, "cost_per_test": 0.046871, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 52.826, "latency": 18.892, "stderr": 3.448, "cost_per_test": 0.013517, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 48.382, "latency": 409.746, "stderr": 3.018, "cost_per_test": 0.004424, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "weighted_pass__business_commercial": { "anthropic/claude-fable-5-1": { "accuracy": 94.617, "latency": 3855.165, "stderr": 1.435, "cost_per_test": 27.230592, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 92.246, "latency": 2163.785, "stderr": 2.569, "cost_per_test": 7.357666, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 91.87, "latency": 2252.454, "stderr": 1.752, "cost_per_test": 7.818685, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 91.526, "latency": 755.219, "stderr": 1.921, "cost_per_test": 0.250157, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 91.374, "latency": 1482.098, "stderr": 1.907, "cost_per_test": 9.515091, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 91.27, "latency": 3410.204, "stderr": 1.939, "cost_per_test": 0.266594, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 91.066, "latency": 315.567, "stderr": 1.922, "cost_per_test": 1.733887, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-4-carbon": { "accuracy": 90.979, "latency": 3764.53, "stderr": 2.033, "cost_per_test": 10.672138, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 89.811, "latency": 1748.44, "stderr": 3.186, "cost_per_test": 3.900906, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 89.75, "latency": 1612.05, "stderr": 2.478, "cost_per_test": 1.54459, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 89.525, "latency": 298.916, "stderr": 2.803, "cost_per_test": 0.773586, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 89.217, "latency": 328.596, "stderr": 2.434, "cost_per_test": 0.506176, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 89.091, "latency": 319.151, "stderr": 2.687, "cost_per_test": 0.363621, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "stepfun/step-5-preview": { "accuracy": 89.077, "latency": 2344.543, "stderr": 2.243, "cost_per_test": 0.811263, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "zai/glm-5.3": { "accuracy": 89.064, "latency": 3261.478, "stderr": 2.74, "cost_per_test": 2.29039, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 89.003, "latency": 4581.547, "stderr": 2.864, "cost_per_test": 19.023356, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-spark-open": { "accuracy": 88.955, "latency": 1523.334, "stderr": 2.648, "cost_per_test": 1.137443, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 88.674, "latency": 353.92, "stderr": 3.166, "cost_per_test": 0.626202, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 88.343, "latency": 626.261, "stderr": 2.835, "cost_per_test": 3.528557, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 88.24, "latency": 581.128, "stderr": 2.861, "cost_per_test": 0.642459, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 87.972, "latency": 1023.821, "stderr": 2.977, "cost_per_test": 2.037941, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 87.785, "latency": 2247.75, "stderr": 2.946, "cost_per_test": 0.705293, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 87.673, "latency": 4069.494, "stderr": 2.959, "cost_per_test": 0.863275, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 87.618, "latency": 2633.008, "stderr": 2.579, "cost_per_test": 0.372136, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "fireworks/ember-1": { "accuracy": 87.601, "latency": 2895.709, "stderr": 2.247, "cost_per_test": 4.184362, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.2": { "accuracy": 87.466, "latency": 1106.364, "stderr": 2.623, "cost_per_test": 0.920113, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.7": { "accuracy": 87.402, "latency": 1380.854, "stderr": 2.776, "cost_per_test": 4.555398, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 87.309, "latency": 2334.81, "stderr": 2.242, "cost_per_test": 0.78052, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 87.024, "latency": 5681.673, "stderr": 3.122, "cost_per_test": 1.820007, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 86.88, "latency": 904.491, "stderr": 3.443, "cost_per_test": 3.808111, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.5": { "accuracy": 86.805, "latency": 913.577, "stderr": 2.755, "cost_per_test": 0.893714, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 86.764, "latency": 2266.219, "stderr": 2.947, "cost_per_test": 6.684872, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 86.759, "latency": 1790.227, "stderr": 2.83, "cost_per_test": 3.031928, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 86.756, "latency": 1723.973, "stderr": 2.963, "cost_per_test": 2.311817, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 86.675, "latency": 694.588, "stderr": 2.424, "cost_per_test": 0.226487, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 86.655, "latency": 3920.461, "stderr": 3.369, "cost_per_test": 2.547702, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 86.491, "latency": 917.178, "stderr": 2.924, "cost_per_test": 0.622362, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 86.246, "latency": 1961.481, "stderr": 2.635, "cost_per_test": 1.153979, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 86.128, "latency": 4084.965, "stderr": 2.961, "cost_per_test": 7.302701, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 86.112, "latency": 777.706, "stderr": 3.565, "cost_per_test": 2.888428, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 86.015, "latency": 5387.125, "stderr": 3.017, "cost_per_test": 1.920252, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 85.024, "latency": 1783.688, "stderr": 3.677, "cost_per_test": 0.170401, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 84.578, "latency": 173.23, "stderr": 3.01, "cost_per_test": 0.487431, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-legal": { "accuracy": 84.545, "latency": 2576.157, "stderr": 3.631, "cost_per_test": 0.05022, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 84.159, "latency": 2357.851, "stderr": 3.13, "cost_per_test": 3.455558, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 84.143, "latency": 317.652, "stderr": 2.921, "cost_per_test": 1.232191, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 84.011, "latency": 2499.931, "stderr": 3.767, "cost_per_test": 1.025341, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 83.821, "latency": 1735.645, "stderr": 3.003, "cost_per_test": 10.624987, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 83.547, "latency": 790.908, "stderr": 3.543, "cost_per_test": 3.239177, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 83.293, "latency": 1087.928, "stderr": 3.436, "cost_per_test": 0.215579, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4": { "accuracy": 82.74, "latency": 5721.537, "stderr": 4.469, "cost_per_test": 2.262179, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 82.694, "latency": 867.939, "stderr": 3.326, "cost_per_test": 0.34538, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "inception/mercury-3": { "accuracy": 82.557, "latency": 498.212, "stderr": 2.852, "cost_per_test": 0.086143, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "zai/glm-5.1": { "accuracy": 81.655, "latency": 336.248, "stderr": 2.98, "cost_per_test": 0.399162, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 81.634, "latency": 909.412, "stderr": 3.932, "cost_per_test": 0.066442, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 81.54, "latency": 663.553, "stderr": 2.354, "cost_per_test": 0.73626, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 81.111, "latency": 452.214, "stderr": 3.408, "cost_per_test": 0.075902, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 80.966, "latency": 913.289, "stderr": 3.37, "cost_per_test": 1.493846, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 80.773, "latency": 216.623, "stderr": 3.085, "cost_per_test": 0.45295, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 80.401, "latency": 1674.291, "stderr": 3.149, "cost_per_test": 5.299121, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 79.089, "latency": 792.476, "stderr": 3.455, "cost_per_test": 0.529635, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-luna": { "accuracy": 79.086, "latency": 2678.679, "stderr": 3.272, "cost_per_test": 0.455548, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 78.436, "latency": 1873.65, "stderr": 3.571, "cost_per_test": 1.432344, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 78.197, "latency": 709.804, "stderr": 4.24, "cost_per_test": 0.132473, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "reflection/Beam-1.0": { "accuracy": 78.141, "latency": 108.067, "stderr": 3.702, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 77.77, "latency": 6273.67, "stderr": 5.246, "cost_per_test": 15.137463, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 76.138, "latency": 787.349, "stderr": 3.499, "cost_per_test": 0.216443, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 75.896, "latency": 5207.222, "stderr": 3.665, "cost_per_test": 3.545917, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 75.874, "latency": 200.727, "stderr": 3.702, "cost_per_test": 0.135638, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 75.839, "latency": 405.023, "stderr": 3.853, "cost_per_test": 0.029283, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.3": { "accuracy": 75.476, "latency": 73.592, "stderr": 3.45, "cost_per_test": 0.170453, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 74.083, "latency": 489.906, "stderr": 3.402, "cost_per_test": 0.032371, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 73.216, "latency": 62.398, "stderr": 3.525, "cost_per_test": 0.120286, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 73.068, "latency": 8276.057, "stderr": 6.186, "cost_per_test": 5.198048, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 71.304, "latency": 637.024, "stderr": 3.21, "cost_per_test": 1.423124, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 71.172, "latency": 2506.739, "stderr": 3.437, "cost_per_test": 2.128528, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 71.001, "latency": 108.913, "stderr": 4.146, "cost_per_test": 0.08423, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 70.961, "latency": 7461.121, "stderr": 6.321, "cost_per_test": 26.294195, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 70.813, "latency": 1493.936, "stderr": 3.808, "cost_per_test": 0.245468, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 70.541, "latency": 620.168, "stderr": 4.223, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 69.927, "latency": 261.305, "stderr": 4.057, "cost_per_test": 0.042315, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 69.871, "latency": 241.456, "stderr": 4.105, "cost_per_test": 0.168225, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 68.944, "latency": 279.601, "stderr": 4.08, "cost_per_test": 0.268062, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 68.55, "latency": 343.417, "stderr": 4.162, "cost_per_test": 0.211419, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 66.395, "latency": 568.266, "stderr": 4.306, "cost_per_test": 0.150807, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 66.266, "latency": 6424.655, "stderr": 6.25, "cost_per_test": 7.199044, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 65.381, "latency": 44.657, "stderr": 3.486, "cost_per_test": 0.057037, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-m.1": { "accuracy": 62.671, "latency": 402.376, "stderr": 4.228, "cost_per_test": 0.204493, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 59.692, "latency": 399.428, "stderr": 3.772, "cost_per_test": 0.12794, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 54.626, "latency": 23.436, "stderr": 3.647, "cost_per_test": 0.014473, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 53.678, "latency": 124.79, "stderr": 3.927, "cost_per_test": 0.004634, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "weighted_pass__civil_litigation": { "google/gemini-4-carbon": { "accuracy": 96.504, "latency": 3262.96, "stderr": 0.992, "cost_per_test": 9.405202, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-4-argon": { "accuracy": 95.082, "latency": 1965.433, "stderr": 1.26, "cost_per_test": 6.770674, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 94.915, "latency": 3586.008, "stderr": 2.241, "cost_per_test": 25.254544, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 92.925, "latency": 1439.733, "stderr": 2.59, "cost_per_test": 3.415131, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 92.791, "latency": 2079.281, "stderr": 2.492, "cost_per_test": 7.304788, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 91.539, "latency": 254.44, "stderr": 2.326, "cost_per_test": 1.35934, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 91.451, "latency": 1489.886, "stderr": 2.697, "cost_per_test": 10.123039, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 91.189, "latency": 557.173, "stderr": 3.626, "cost_per_test": 0.757486, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 91.057, "latency": 1539.561, "stderr": 2.905, "cost_per_test": 1.681204, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 90.891, "latency": 305.909, "stderr": 3.793, "cost_per_test": 0.581163, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-legal": { "accuracy": 90.874, "latency": 2608.688, "stderr": 2.47, "cost_per_test": 0.055323, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 90.836, "latency": 3900.655, "stderr": 2.889, "cost_per_test": 0.854674, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 90.753, "latency": 806.663, "stderr": 2.985, "cost_per_test": 2.997652, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 90.695, "latency": 2417.44, "stderr": 3.073, "cost_per_test": 0.338953, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 90.225, "latency": 3816.903, "stderr": 2.781, "cost_per_test": 18.560919, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 89.997, "latency": 1439.49, "stderr": 2.704, "cost_per_test": 5.606604, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 89.995, "latency": 2136.958, "stderr": 2.765, "cost_per_test": 0.917209, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "zai/glm-5.3": { "accuracy": 89.911, "latency": 2589.489, "stderr": 3.291, "cost_per_test": 2.051403, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 89.586, "latency": 3261.102, "stderr": 3.159, "cost_per_test": 0.259741, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4": { "accuracy": 89.309, "latency": 5187.901, "stderr": 3.946, "cost_per_test": 2.080453, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse-wonder": { "accuracy": 89.018, "latency": 885.474, "stderr": 3.3, "cost_per_test": 4.274558, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 88.953, "latency": 308.847, "stderr": 3.252, "cost_per_test": 0.367043, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 88.914, "latency": 288.591, "stderr": 2.93, "cost_per_test": 0.719234, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 88.748, "latency": 1225.988, "stderr": 2.752, "cost_per_test": 1.215179, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 88.718, "latency": 1956.425, "stderr": 2.99, "cost_per_test": 3.400541, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 88.711, "latency": 1104.503, "stderr": 2.935, "cost_per_test": 0.246003, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 88.669, "latency": 1743.79, "stderr": 2.508, "cost_per_test": 13.735474, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 88.446, "latency": 1554.157, "stderr": 2.957, "cost_per_test": 1.138521, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 88.404, "latency": 744.696, "stderr": 4.311, "cost_per_test": 0.267966, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse-spark-open": { "accuracy": 88.253, "latency": 1273.914, "stderr": 3.238, "cost_per_test": 1.590079, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 88.182, "latency": 538.692, "stderr": 3.314, "cost_per_test": 2.902212, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 87.708, "latency": 2447.426, "stderr": 2.161, "cost_per_test": 3.801126, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 87.667, "latency": 2240.6, "stderr": 3.339, "cost_per_test": 0.207888, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.5": { "accuracy": 87.616, "latency": 2408.394, "stderr": 3.561, "cost_per_test": 7.00366, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 87.539, "latency": 3932.081, "stderr": 3.014, "cost_per_test": 7.563297, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 87.434, "latency": 1001.064, "stderr": 2.883, "cost_per_test": 2.550366, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 87.42, "latency": 304.101, "stderr": 3.571, "cost_per_test": 0.602276, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "fireworks/ember-1": { "accuracy": 87.147, "latency": 2689.198, "stderr": 3.087, "cost_per_test": 4.546668, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 87.099, "latency": 663.732, "stderr": 3.706, "cost_per_test": 0.239442, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 86.955, "latency": 885.063, "stderr": 3.446, "cost_per_test": 0.857222, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 86.308, "latency": 255.111, "stderr": 2.832, "cost_per_test": 1.230964, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 85.96, "latency": 1095.538, "stderr": 3.534, "cost_per_test": 0.736017, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 85.824, "latency": 5842.182, "stderr": 3.368, "cost_per_test": 2.843836, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 85.785, "latency": 2280.787, "stderr": 3.712, "cost_per_test": 1.099633, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 85.331, "latency": 192.418, "stderr": 2.941, "cost_per_test": 0.477549, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling": { "accuracy": 84.98, "latency": 740.554, "stderr": 3.193, "cost_per_test": 0.559142, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 84.854, "latency": 823.116, "stderr": 2.956, "cost_per_test": 0.385069, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 84.852, "latency": 2088.431, "stderr": 4.591, "cost_per_test": 0.658074, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 84.826, "latency": 2149.371, "stderr": 3.355, "cost_per_test": 0.78097, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-3": { "accuracy": 84.638, "latency": 493.563, "stderr": 3.616, "cost_per_test": 0.076555, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 84.566, "latency": 876.732, "stderr": 3.007, "cost_per_test": 1.504309, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 84.186, "latency": 3478.586, "stderr": 4.031, "cost_per_test": 2.299147, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k3": { "accuracy": 83.776, "latency": 838.559, "stderr": 3.96, "cost_per_test": 4.229889, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.1": { "accuracy": 83.513, "latency": 361.704, "stderr": 3.846, "cost_per_test": 0.473442, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 83.098, "latency": 1304.611, "stderr": 3.939, "cost_per_test": 1.737788, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 83.039, "latency": 5529.804, "stderr": 5.588, "cost_per_test": 1.750979, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 82.829, "latency": 840.078, "stderr": 4.722, "cost_per_test": 0.059764, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 82.707, "latency": 225.619, "stderr": 2.966, "cost_per_test": 0.508937, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "reflection/Beam-1.0": { "accuracy": 81.857, "latency": 92.542, "stderr": 2.907, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 81.699, "latency": 6231.017, "stderr": 6.129, "cost_per_test": 15.071116, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 81.523, "latency": 676.765, "stderr": 4.293, "cost_per_test": 0.13253, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 81.331, "latency": 775.441, "stderr": 3.74, "cost_per_test": 0.228401, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 81.028, "latency": 7758.778, "stderr": 6.771, "cost_per_test": 5.531551, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 80.907, "latency": 486.481, "stderr": 4.255, "cost_per_test": 0.080617, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 80.55, "latency": 4095.85, "stderr": 2.928, "cost_per_test": 3.018691, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 80.484, "latency": 478.114, "stderr": 3.245, "cost_per_test": 0.031614, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-6-luna": { "accuracy": 78.633, "latency": 2593.828, "stderr": 4.852, "cost_per_test": 0.46119, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 78.225, "latency": 89.721, "stderr": 5.121, "cost_per_test": 0.06691, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "openai/gpt-6-sol": { "accuracy": 77.964, "latency": 1441.302, "stderr": 4.184, "cost_per_test": 5.211283, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 77.766, "latency": 246.451, "stderr": 3.905, "cost_per_test": 0.200307, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 77.536, "latency": 698.535, "stderr": 5.049, "cost_per_test": 0.632609, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 77.196, "latency": 1372.833, "stderr": 4.487, "cost_per_test": 0.220822, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 77.014, "latency": 215.912, "stderr": 4.021, "cost_per_test": 0.132078, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 76.085, "latency": 186.413, "stderr": 4.453, "cost_per_test": 0.040156, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 75.83, "latency": 6255.777, "stderr": 6.713, "cost_per_test": 7.220094, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 75.8, "latency": 1733.848, "stderr": 4.147, "cost_per_test": 1.440762, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 74.02, "latency": 6638.643, "stderr": 7.86, "cost_per_test": 22.569939, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 71.791, "latency": 428.141, "stderr": 5.165, "cost_per_test": 0.242453, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 71.755, "latency": 416.27, "stderr": 5.091, "cost_per_test": 0.037073, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 71.52, "latency": 690.614, "stderr": 5.401, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 70.628, "latency": 277.344, "stderr": 4.672, "cost_per_test": 0.287416, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 68.852, "latency": 68.656, "stderr": 4.95, "cost_per_test": 0.19461, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 68.638, "latency": 725.151, "stderr": 4.518, "cost_per_test": 0.180106, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 68.21, "latency": 56.454, "stderr": 5.443, "cost_per_test": 0.108873, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 67.267, "latency": 2120.497, "stderr": 5.492, "cost_per_test": 1.875076, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 67.094, "latency": 450.642, "stderr": 4.204, "cost_per_test": 0.229555, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 65.952, "latency": 595.409, "stderr": 5.685, "cost_per_test": 1.316462, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 65.55, "latency": 47.033, "stderr": 5.36, "cost_per_test": 0.064712, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 58.849, "latency": 19.793, "stderr": 5.219, "cost_per_test": 0.013273, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 58.132, "latency": 332.669, "stderr": 5.923, "cost_per_test": 0.119391, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 48.511, "latency": 273.794, "stderr": 5.134, "cost_per_test": 0.004442, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "weighted_pass__constitutional_civil_rights": { "google/gemini-4-carbon": { "accuracy": 91.233, "latency": 2436.37, "stderr": 2.13, "cost_per_test": 6.991706, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 90.702, "latency": 1151.764, "stderr": 2.027, "cost_per_test": 2.721277, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 90.334, "latency": 1361.337, "stderr": 2.285, "cost_per_test": 4.353414, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 89.868, "latency": 3510.897, "stderr": 2.092, "cost_per_test": 21.64278, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 89.259, "latency": 632.775, "stderr": 2.032, "cost_per_test": 2.959097, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-spark-open": { "accuracy": 89.18, "latency": 1835.592, "stderr": 2.515, "cost_per_test": 0.898996, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-5.3": { "accuracy": 88.98, "latency": 2583.919, "stderr": 2.787, "cost_per_test": 1.779991, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 88.843, "latency": 2446.246, "stderr": 2.289, "cost_per_test": 0.179145, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 88.662, "latency": 645.429, "stderr": 2.631, "cost_per_test": 0.225678, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 88.587, "latency": 1898.737, "stderr": 2.721, "cost_per_test": 6.396226, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 88.31, "latency": 3868.674, "stderr": 1.827, "cost_per_test": 14.526185, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 88.222, "latency": 1396.958, "stderr": 2.316, "cost_per_test": 1.277781, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 88.144, "latency": 220.844, "stderr": 2.458, "cost_per_test": 1.043129, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 88.07, "latency": 4751.209, "stderr": 2.389, "cost_per_test": 2.624566, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 87.91, "latency": 977.356, "stderr": 2.549, "cost_per_test": 6.898245, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 87.871, "latency": 853.113, "stderr": 2.478, "cost_per_test": 1.965994, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-legal": { "accuracy": 87.634, "latency": 2063.284, "stderr": 2.771, "cost_per_test": 0.037714, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 87.039, "latency": 1566.47, "stderr": 2.548, "cost_per_test": 2.477637, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 87.016, "latency": 609.116, "stderr": 2.634, "cost_per_test": 2.462184, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 86.909, "latency": 1461.512, "stderr": 2.637, "cost_per_test": 0.943808, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 86.57, "latency": 1675.051, "stderr": 2.54, "cost_per_test": 0.53415, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "fireworks/ember-1": { "accuracy": 86.503, "latency": 1748.11, "stderr": 2.43, "cost_per_test": 2.722205, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 86.439, "latency": 5827.977, "stderr": 2.943, "cost_per_test": 14.117789, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 86.153, "latency": 1718.661, "stderr": 2.327, "cost_per_test": 0.583776, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 86.141, "latency": 1049.066, "stderr": 2.978, "cost_per_test": 4.00044, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.2": { "accuracy": 86.095, "latency": 558.667, "stderr": 2.502, "cost_per_test": 0.395873, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.5": { "accuracy": 85.897, "latency": 907.475, "stderr": 2.679, "cost_per_test": 0.836603, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 85.749, "latency": 1785.016, "stderr": 2.782, "cost_per_test": 0.721618, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 85.744, "latency": 1096.62, "stderr": 2.309, "cost_per_test": 0.614384, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 85.714, "latency": 518.106, "stderr": 2.752, "cost_per_test": 0.191689, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 85.619, "latency": 310.439, "stderr": 2.661, "cost_per_test": 0.303515, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 85.223, "latency": 224.844, "stderr": 2.655, "cost_per_test": 0.987781, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 85.201, "latency": 3263.997, "stderr": 3.337, "cost_per_test": 2.246783, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 85.08, "latency": 733.352, "stderr": 2.727, "cost_per_test": 0.291222, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 85.04, "latency": 6953.006, "stderr": 4.021, "cost_per_test": 4.449226, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 85.038, "latency": 6438.62, "stderr": 3.871, "cost_per_test": 22.403219, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 84.883, "latency": 355.395, "stderr": 2.734, "cost_per_test": 0.362114, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 84.836, "latency": 572.241, "stderr": 2.937, "cost_per_test": 2.490256, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 84.731, "latency": 1025.06, "stderr": 2.991, "cost_per_test": 0.204391, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 84.701, "latency": 1943.878, "stderr": 3.332, "cost_per_test": 0.828943, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 84.694, "latency": 275.608, "stderr": 4.064, "cost_per_test": 0.430216, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 84.322, "latency": 1681.88, "stderr": 3.584, "cost_per_test": 0.17967, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 84.319, "latency": 2858.564, "stderr": 3.335, "cost_per_test": 0.671944, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 84.16, "latency": 261.358, "stderr": 4.175, "cost_per_test": 0.485181, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 84.082, "latency": 214.833, "stderr": 2.738, "cost_per_test": 0.667444, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 84.029, "latency": 427.276, "stderr": 3.803, "cost_per_test": 2.058545, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 83.985, "latency": 1775.958, "stderr": 2.699, "cost_per_test": 2.692913, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 83.745, "latency": 2156.642, "stderr": 2.425, "cost_per_test": 5.731692, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 83.29, "latency": 522.986, "stderr": 4.048, "cost_per_test": 0.545149, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 83.156, "latency": 160.664, "stderr": 2.78, "cost_per_test": 0.393807, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 82.968, "latency": 3655.679, "stderr": 2.866, "cost_per_test": 6.783177, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 82.74, "latency": 611.367, "stderr": 3.508, "cost_per_test": 0.372278, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 82.728, "latency": 2834.861, "stderr": 3.541, "cost_per_test": 0.421978, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 82.71, "latency": 828.518, "stderr": 2.708, "cost_per_test": 0.208128, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-astra": { "accuracy": 82.609, "latency": 1280.117, "stderr": 2.564, "cost_per_test": 8.37428, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 82.374, "latency": 303.387, "stderr": 3.028, "cost_per_test": 0.019445, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 82.294, "latency": 5507.901, "stderr": 3.843, "cost_per_test": 1.55544, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 82.248, "latency": 782.498, "stderr": 3.483, "cost_per_test": 1.136745, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 82.161, "latency": 384.09, "stderr": 3.376, "cost_per_test": 0.079592, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 82.013, "latency": 1286.448, "stderr": 3.51, "cost_per_test": 1.563366, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "tencent/hy4": { "accuracy": 81.8, "latency": 4486.623, "stderr": 4.575, "cost_per_test": 1.742342, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "inception/mercury-3": { "accuracy": 81.271, "latency": 353.782, "stderr": 3.674, "cost_per_test": 0.052994, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 80.858, "latency": 589.79, "stderr": 3.171, "cost_per_test": 0.111862, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "reflection/Beam-1.0": { "accuracy": 80.153, "latency": 62.09, "stderr": 3.56, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 79.637, "latency": 223.214, "stderr": 2.872, "cost_per_test": 0.466165, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 79.224, "latency": 1564.212, "stderr": 3.12, "cost_per_test": 1.25835, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 78.962, "latency": 830.725, "stderr": 4.476, "cost_per_test": 0.076515, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 78.706, "latency": 696.299, "stderr": 3.142, "cost_per_test": 0.516518, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 78.044, "latency": 214.461, "stderr": 2.715, "cost_per_test": 0.119798, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 77.981, "latency": 3732.629, "stderr": 3.581, "cost_per_test": 2.673236, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 77.97, "latency": 1202.973, "stderr": 3.132, "cost_per_test": 4.442219, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 77.484, "latency": 61.408, "stderr": 3.778, "cost_per_test": 0.080105, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 77.373, "latency": 1958.935, "stderr": 4.237, "cost_per_test": 0.364867, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 76.763, "latency": 74.98, "stderr": 3.949, "cost_per_test": 0.047883, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 75.784, "latency": 352.653, "stderr": 3.82, "cost_per_test": 0.029482, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.3": { "accuracy": 74.824, "latency": 125.212, "stderr": 3.27, "cost_per_test": 0.250093, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 74.537, "latency": 6612.654, "stderr": 5.33, "cost_per_test": 7.25464, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 74.474, "latency": 276.602, "stderr": 4.007, "cost_per_test": 0.260695, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 74.368, "latency": 1326.72, "stderr": 3.648, "cost_per_test": 0.216972, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 74.113, "latency": 1433.795, "stderr": 3.484, "cost_per_test": 1.467472, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 73.093, "latency": 279.901, "stderr": 3.814, "cost_per_test": 0.255064, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 72.275, "latency": 176.967, "stderr": 4.471, "cost_per_test": 0.030415, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "poolside/laguna-m.1": { "accuracy": 70.15, "latency": 540.649, "stderr": 3.678, "cost_per_test": 0.230889, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 69.567, "latency": 449.168, "stderr": 4.186, "cost_per_test": 0.209051, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 69.251, "latency": 822.903, "stderr": 4.888, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 68.181, "latency": 715.501, "stderr": 4.487, "cost_per_test": 1.425655, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 65.258, "latency": 597.509, "stderr": 4.328, "cost_per_test": 0.134636, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 63.691, "latency": 293.416, "stderr": 4.815, "cost_per_test": 0.090182, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "inception/mercury-2.5": { "accuracy": 62.892, "latency": 53.037, "stderr": 4.365, "cost_per_test": 0.058844, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 61.883, "latency": 21.603, "stderr": 4.315, "cost_per_test": 0.013343, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 53.161, "latency": 268.813, "stderr": 4.863, "cost_per_test": 0.003527, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "weighted_pass__criminal": { "google/gemini-4-carbon": { "accuracy": 90.59, "latency": 3164.955, "stderr": 2.11, "cost_per_test": 8.842662, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-4-argon": { "accuracy": 90.142, "latency": 1907.88, "stderr": 2.078, "cost_per_test": 6.616583, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 89.439, "latency": 1527.247, "stderr": 2.129, "cost_per_test": 3.617802, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 88.777, "latency": 315.871, "stderr": 2.213, "cost_per_test": 1.534611, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 88.075, "latency": 3466.552, "stderr": 2.124, "cost_per_test": 22.326482, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 88.073, "latency": 2118.512, "stderr": 2.29, "cost_per_test": 6.903486, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 87.949, "latency": 1371.988, "stderr": 2.265, "cost_per_test": 5.268708, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 87.747, "latency": 755.757, "stderr": 2.443, "cost_per_test": 2.852274, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder": { "accuracy": 87.417, "latency": 967.654, "stderr": 2.225, "cost_per_test": 4.059119, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 87.238, "latency": 2957.873, "stderr": 2.506, "cost_per_test": 2.182135, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.6": { "accuracy": 87.179, "latency": 1356.005, "stderr": 2.392, "cost_per_test": 1.460147, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 87.174, "latency": 1928.013, "stderr": 2.313, "cost_per_test": 0.177786, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 86.657, "latency": 303.011, "stderr": 2.762, "cost_per_test": 0.60538, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 86.381, "latency": 4987.986, "stderr": 2.307, "cost_per_test": 24.12378, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 86.134, "latency": 1398.537, "stderr": 2.332, "cost_per_test": 9.468125, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 85.936, "latency": 2749.369, "stderr": 2.265, "cost_per_test": 0.250408, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 85.796, "latency": 989.26, "stderr": 2.204, "cost_per_test": 1.99149, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 85.453, "latency": 2183.678, "stderr": 2.39, "cost_per_test": 2.568801, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 85.445, "latency": 320.962, "stderr": 2.33, "cost_per_test": 0.345173, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 84.941, "latency": 3997.963, "stderr": 2.401, "cost_per_test": 2.676114, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k3": { "accuracy": 84.643, "latency": 772.67, "stderr": 2.462, "cost_per_test": 3.543085, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 84.532, "latency": 767.942, "stderr": 2.949, "cost_per_test": 0.321616, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 84.458, "latency": 2341.734, "stderr": 2.235, "cost_per_test": 3.38256, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 84.284, "latency": 347.571, "stderr": 2.652, "cost_per_test": 0.827264, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 84.249, "latency": 6259.782, "stderr": 2.412, "cost_per_test": 14.876209, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 84.145, "latency": 2210.107, "stderr": 2.586, "cost_per_test": 0.87429, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 84.113, "latency": 540.381, "stderr": 2.952, "cost_per_test": 0.616709, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 84.06, "latency": 1506.644, "stderr": 2.665, "cost_per_test": 2.518676, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-legal": { "accuracy": 83.857, "latency": 2208.959, "stderr": 2.777, "cost_per_test": 0.046529, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 83.849, "latency": 5566.578, "stderr": 2.561, "cost_per_test": 1.725041, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 83.596, "latency": 2498.94, "stderr": 2.455, "cost_per_test": 1.017486, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 83.51, "latency": 5432.346, "stderr": 2.872, "cost_per_test": 3.124013, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 83.209, "latency": 2224.103, "stderr": 2.854, "cost_per_test": 0.715843, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 83.205, "latency": 1658.26, "stderr": 2.276, "cost_per_test": 10.941763, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 83.127, "latency": 301.211, "stderr": 3.178, "cost_per_test": 0.446419, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 82.909, "latency": 7162.351, "stderr": 3.481, "cost_per_test": 25.272629, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 82.619, "latency": 874.891, "stderr": 2.863, "cost_per_test": 0.822469, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "tencent/hy4": { "accuracy": 82.584, "latency": 5104.912, "stderr": 3.195, "cost_per_test": 2.104097, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "fireworks/ember-1": { "accuracy": 82.57, "latency": 2079.561, "stderr": 2.465, "cost_per_test": 3.601082, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 82.561, "latency": 4543.691, "stderr": 2.55, "cost_per_test": 8.912211, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 82.44, "latency": 256.075, "stderr": 2.626, "cost_per_test": 1.359179, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 82.222, "latency": 2563.313, "stderr": 2.562, "cost_per_test": 0.994665, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 82.029, "latency": 529.681, "stderr": 2.544, "cost_per_test": 2.779258, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 81.944, "latency": 7534.426, "stderr": 3.778, "cost_per_test": 5.274524, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.5": { "accuracy": 81.837, "latency": 2310.474, "stderr": 2.851, "cost_per_test": 7.322657, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 81.761, "latency": 3744.173, "stderr": 2.971, "cost_per_test": 0.819373, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "inception/mercury-3": { "accuracy": 81.641, "latency": 425.103, "stderr": 2.748, "cost_per_test": 0.06767, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 81.349, "latency": 1712.774, "stderr": 2.933, "cost_per_test": 1.169956, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 81.01, "latency": 189.608, "stderr": 2.666, "cost_per_test": 0.501214, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.2": { "accuracy": 80.073, "latency": 1034.653, "stderr": 2.733, "cost_per_test": 0.846458, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 79.928, "latency": 2805.101, "stderr": 2.964, "cost_per_test": 0.425544, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-sol": { "accuracy": 79.693, "latency": 1562.595, "stderr": 2.698, "cost_per_test": 5.495004, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 79.608, "latency": 1433.199, "stderr": 2.988, "cost_per_test": 1.995877, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 79.256, "latency": 682.99, "stderr": 3.087, "cost_per_test": 0.233636, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 79.004, "latency": 2463.873, "stderr": 3.058, "cost_per_test": 0.454694, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 78.143, "latency": 331.685, "stderr": 3.194, "cost_per_test": 0.405872, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 78.11, "latency": 855.631, "stderr": 2.891, "cost_per_test": 0.325282, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "reflection/Beam-1.0": { "accuracy": 78.109, "latency": 87.152, "stderr": 2.943, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 78.032, "latency": 711.836, "stderr": 2.961, "cost_per_test": 0.143249, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 77.693, "latency": 787.984, "stderr": 2.824, "cost_per_test": 1.332257, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 77.42, "latency": 872.79, "stderr": 2.901, "cost_per_test": 0.509253, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 76.836, "latency": 398.106, "stderr": 2.757, "cost_per_test": 0.081998, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 76.823, "latency": 1264.855, "stderr": 3.003, "cost_per_test": 0.25224, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 76.39, "latency": 1004.085, "stderr": 2.887, "cost_per_test": 0.615927, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 76.229, "latency": 4308.911, "stderr": 2.992, "cost_per_test": 3.298261, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 76.177, "latency": 217.466, "stderr": 2.892, "cost_per_test": 0.48486, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 75.057, "latency": 1937.218, "stderr": 2.901, "cost_per_test": 1.608029, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 74.773, "latency": 435.186, "stderr": 3.265, "cost_per_test": 0.022099, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 74.684, "latency": 1025.226, "stderr": 3.856, "cost_per_test": 0.071477, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 73.1, "latency": 858.129, "stderr": 3.258, "cost_per_test": 0.2621, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 72.798, "latency": 1398.791, "stderr": 3.182, "cost_per_test": 0.25486, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 72.137, "latency": 221.166, "stderr": 2.88, "cost_per_test": 0.13559, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 71.257, "latency": 78.654, "stderr": 3.4, "cost_per_test": 0.062754, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 69.682, "latency": 6680.114, "stderr": 4.745, "cost_per_test": 8.328752, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.3": { "accuracy": 69.316, "latency": 110.22, "stderr": 3.232, "cost_per_test": 0.199087, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 68.802, "latency": 698.746, "stderr": 3.45, "cost_per_test": 0.68798, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 68.108, "latency": 63.499, "stderr": 3.364, "cost_per_test": 0.122864, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 67.541, "latency": 394.953, "stderr": 3.311, "cost_per_test": 0.023143, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 67.321, "latency": 267.815, "stderr": 3.459, "cost_per_test": 0.155562, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 66.155, "latency": 651.134, "stderr": 3.844, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 64.416, "latency": 283.18, "stderr": 3.525, "cost_per_test": 0.260687, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 64.389, "latency": 2330.292, "stderr": 3.396, "cost_per_test": 2.024329, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 63.41, "latency": 421.649, "stderr": 3.619, "cost_per_test": 0.226204, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 62.879, "latency": 185.804, "stderr": 3.646, "cost_per_test": 0.034171, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "inception/mercury-2.5": { "accuracy": 61.813, "latency": 48.522, "stderr": 3.438, "cost_per_test": 0.059734, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-m.1": { "accuracy": 61.647, "latency": 521.807, "stderr": 3.495, "cost_per_test": 0.26163, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 60.899, "latency": 611.807, "stderr": 3.853, "cost_per_test": 1.490522, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 57.92, "latency": 548.936, "stderr": 3.278, "cost_per_test": 0.153371, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 56.943, "latency": 20.102, "stderr": 3.825, "cost_per_test": 0.013186, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 56.382, "latency": 301.868, "stderr": 3.95, "cost_per_test": 0.108798, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 51.248, "latency": 187.227, "stderr": 3.656, "cost_per_test": 0.003839, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "weighted_pass__family": { "anthropic/claude-opus-5": { "accuracy": 86.646, "latency": 1858.312, "stderr": 3.881, "cost_per_test": 6.195885, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-carbon": { "accuracy": 84.761, "latency": 4551.243, "stderr": 4.625, "cost_per_test": 12.648487, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 84.308, "latency": 357.472, "stderr": 4.498, "cost_per_test": 0.633898, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 83.42, "latency": 1437.652, "stderr": 4.044, "cost_per_test": 5.323844, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 83.048, "latency": 2250.577, "stderr": 4.509, "cost_per_test": 4.730031, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 81.509, "latency": 2821.52, "stderr": 4.4, "cost_per_test": 10.91868, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 80.388, "latency": 1405.167, "stderr": 3.936, "cost_per_test": 1.425494, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-legal": { "accuracy": 80.107, "latency": 2827.87, "stderr": 5.612, "cost_per_test": 0.051257, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 79.482, "latency": 855.894, "stderr": 3.788, "cost_per_test": 0.383913, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 79.323, "latency": 5063.727, "stderr": 3.94, "cost_per_test": 9.812737, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 79.009, "latency": 716.66, "stderr": 4.403, "cost_per_test": 0.822081, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 78.4, "latency": 3017.714, "stderr": 4.161, "cost_per_test": 1.168665, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 78.125, "latency": 3202.004, "stderr": 5.529, "cost_per_test": 0.286053, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 77.897, "latency": 1643.011, "stderr": 5.851, "cost_per_test": 11.011763, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 77.732, "latency": 1013.398, "stderr": 4.697, "cost_per_test": 0.323772, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 77.576, "latency": 3849.877, "stderr": 5.756, "cost_per_test": 2.697912, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.7": { "accuracy": 77.545, "latency": 1482.157, "stderr": 4.578, "cost_per_test": 5.56694, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 77.423, "latency": 2649.862, "stderr": 5.609, "cost_per_test": 0.231872, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 77.415, "latency": 3235.665, "stderr": 6.068, "cost_per_test": 21.529237, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 77.323, "latency": 787.574, "stderr": 5.197, "cost_per_test": 2.857814, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 77.307, "latency": 1896.183, "stderr": 3.995, "cost_per_test": 15.003346, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 77.267, "latency": 3456.123, "stderr": 4.381, "cost_per_test": 1.536833, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 77.072, "latency": 5320.428, "stderr": 4.982, "cost_per_test": 12.577804, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 76.751, "latency": 1189.264, "stderr": 4.6, "cost_per_test": 2.408033, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 76.736, "latency": 6569.663, "stderr": 4.515, "cost_per_test": 30.084962, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 76.37, "latency": 8086.246, "stderr": 6.633, "cost_per_test": 5.806978, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 76.357, "latency": 6312.514, "stderr": 4.952, "cost_per_test": 3.697863, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 76.317, "latency": 638.032, "stderr": 4.805, "cost_per_test": 3.584147, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 75.432, "latency": 4757.56, "stderr": 5.053, "cost_per_test": 3.063697, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/ember-1": { "accuracy": 75.4, "latency": 3035.777, "stderr": 5.835, "cost_per_test": 5.439212, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 75.195, "latency": 4670.091, "stderr": 5.246, "cost_per_test": 0.970682, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 75.03, "latency": 613.145, "stderr": 6.035, "cost_per_test": 2.522005, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 74.889, "latency": 306.75, "stderr": 4.464, "cost_per_test": 1.594003, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 74.466, "latency": 328.141, "stderr": 5.024, "cost_per_test": 0.452803, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 74.285, "latency": 1762.575, "stderr": 4.898, "cost_per_test": 0.33418, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 74.191, "latency": 2445.946, "stderr": 5.801, "cost_per_test": 0.826784, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 74.056, "latency": 5152.106, "stderr": 5.192, "cost_per_test": 1.724456, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 73.686, "latency": 348.059, "stderr": 4.833, "cost_per_test": 0.406411, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 73.433, "latency": 2357.557, "stderr": 4.48, "cost_per_test": 3.772956, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4": { "accuracy": 73.24, "latency": 6801.677, "stderr": 5.577, "cost_per_test": 2.756433, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse-spark-open": { "accuracy": 72.796, "latency": 2019.893, "stderr": 5.301, "cost_per_test": 0.418234, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 72.424, "latency": 966.329, "stderr": 5.56, "cost_per_test": 4.336354, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 72.268, "latency": 2332.251, "stderr": 4.968, "cost_per_test": 1.856297, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 72.233, "latency": 1564.903, "stderr": 5.607, "cost_per_test": 2.758903, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 71.694, "latency": 1101.097, "stderr": 5.673, "cost_per_test": 0.562977, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-luna": { "accuracy": 71.523, "latency": 2917.375, "stderr": 4.983, "cost_per_test": 0.465031, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 71.357, "latency": 3983.979, "stderr": 5.35, "cost_per_test": 0.672496, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 70.901, "latency": 1329.086, "stderr": 5.57, "cost_per_test": 0.070993, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.5": { "accuracy": 70.836, "latency": 2680.585, "stderr": 4.872, "cost_per_test": 7.622096, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 70.56, "latency": 839.919, "stderr": 5.33, "cost_per_test": 0.991333, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 70.504, "latency": 2391.966, "stderr": 5.948, "cost_per_test": 0.88974, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "reflection/Beam-1.0": { "accuracy": 70.349, "latency": 121.037, "stderr": 5.333, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "zai/glm-5.2": { "accuracy": 69.315, "latency": 1806.747, "stderr": 6.228, "cost_per_test": 1.50639, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 69.188, "latency": 390.844, "stderr": 6.004, "cost_per_test": 0.986437, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 69.17, "latency": 1713.026, "stderr": 6.215, "cost_per_test": 2.549692, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-sol": { "accuracy": 69.065, "latency": 1758.115, "stderr": 5.533, "cost_per_test": 5.389618, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 68.892, "latency": 1001.225, "stderr": 4.477, "cost_per_test": 0.173654, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 68.039, "latency": 951.192, "stderr": 5.235, "cost_per_test": 0.370092, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "inception/mercury-3": { "accuracy": 67.53, "latency": 514.571, "stderr": 6.551, "cost_per_test": 0.08758, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 66.252, "latency": 1317.912, "stderr": 5.618, "cost_per_test": 0.816394, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 66.028, "latency": 7441.78, "stderr": 7.454, "cost_per_test": 25.835741, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 64.806, "latency": 4690.594, "stderr": 5.881, "cost_per_test": 3.506565, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 63.564, "latency": 6905.045, "stderr": 8.026, "cost_per_test": 8.216145, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 62.107, "latency": 167.284, "stderr": 6.045, "cost_per_test": 0.168183, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 61.607, "latency": 1722.37, "stderr": 6.713, "cost_per_test": 0.30233, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 60.959, "latency": 2863.795, "stderr": 6.121, "cost_per_test": 2.335677, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 59.646, "latency": 727.744, "stderr": 6.04, "cost_per_test": 1.001303, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 59.585, "latency": 195.123, "stderr": 6.152, "cost_per_test": 0.581541, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 59.268, "latency": 900.57, "stderr": 6.671, "cost_per_test": 1.420719, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 58.608, "latency": 1055.917, "stderr": 6.891, "cost_per_test": 0.443026, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 58.209, "latency": 434.377, "stderr": 6.265, "cost_per_test": 4.138935, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 58.154, "latency": 342.126, "stderr": 5.699, "cost_per_test": 0.41232, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 58.118, "latency": 435.98, "stderr": 5.723, "cost_per_test": 0.075731, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 53.768, "latency": 98.358, "stderr": 6.267, "cost_per_test": 0.074199, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 52.649, "latency": 422.665, "stderr": 6.426, "cost_per_test": 0.030982, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 51.571, "latency": 692.635, "stderr": 6.939, "cost_per_test": 0.039086, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 51.57, "latency": 4205.69, "stderr": 5.858, "cost_per_test": 2.933958, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 51.562, "latency": 458.965, "stderr": 6.465, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 50.406, "latency": 250.897, "stderr": 6.917, "cost_per_test": 0.288946, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 49.711, "latency": 62.256, "stderr": 6.034, "cost_per_test": 0.169797, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 47.346, "latency": 86.589, "stderr": 6.258, "cost_per_test": 0.230167, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 47.229, "latency": 222.566, "stderr": 6.364, "cost_per_test": 0.036939, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 45.767, "latency": 436.535, "stderr": 6.323, "cost_per_test": 0.255715, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 43.293, "latency": 592.178, "stderr": 6.228, "cost_per_test": 0.140683, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 43.161, "latency": 246.131, "stderr": 5.728, "cost_per_test": 0.16395, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 42.725, "latency": 570.598, "stderr": 5.244, "cost_per_test": 1.090579, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 41.993, "latency": 532.251, "stderr": 5.847, "cost_per_test": 0.261764, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 40.393, "latency": 311.615, "stderr": 5.049, "cost_per_test": 0.005312, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 37.561, "latency": 47.19, "stderr": 5.479, "cost_per_test": 0.062993, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 36.546, "latency": 18.535, "stderr": 5.492, "cost_per_test": 0.013412, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 30.22, "latency": 383.551, "stderr": 5.051, "cost_per_test": 0.13236, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } }, "weighted_pass__health": { "openai/gpt-5.6-sol": { "accuracy": 97.511, "latency": 4151.253, "stderr": 1.194, "cost_per_test": 22.836198, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 96.79, "latency": 1346.948, "stderr": 1.387, "cost_per_test": 7.009258, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-carbon": { "accuracy": 96.764, "latency": 2450.538, "stderr": 1.293, "cost_per_test": 6.751201, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 96.691, "latency": 1693.184, "stderr": 1.324, "cost_per_test": 1.828426, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-wonder": { "accuracy": 96.459, "latency": 858.216, "stderr": 1.845, "cost_per_test": 4.579894, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 96.433, "latency": 4630.853, "stderr": 1.922, "cost_per_test": 12.659413, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 96.193, "latency": 2538.105, "stderr": 1.602, "cost_per_test": 4.232271, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 96.178, "latency": 1923.984, "stderr": 1.399, "cost_per_test": 0.708547, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 96.166, "latency": 3496.535, "stderr": 1.354, "cost_per_test": 2.5406, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-4-argon": { "accuracy": 96.042, "latency": 1338.036, "stderr": 1.567, "cost_per_test": 4.625213, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 95.75, "latency": 1838.285, "stderr": 1.935, "cost_per_test": 3.207198, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 95.589, "latency": 710.57, "stderr": 1.643, "cost_per_test": 0.327988, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.7": { "accuracy": 95.533, "latency": 1095.281, "stderr": 1.692, "cost_per_test": 4.723349, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 95.5, "latency": 1414.958, "stderr": 1.758, "cost_per_test": 3.858235, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 95.48, "latency": 3604.391, "stderr": 1.779, "cost_per_test": 0.88574, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3": { "accuracy": 95.169, "latency": 3215.749, "stderr": 2.39, "cost_per_test": 2.470231, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 95.001, "latency": 5675.961, "stderr": 3.385, "cost_per_test": 20.596615, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 94.942, "latency": 1360.302, "stderr": 1.89, "cost_per_test": 0.193071, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 94.925, "latency": 706.355, "stderr": 2.036, "cost_per_test": 0.272236, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 94.899, "latency": 1553.826, "stderr": 2.096, "cost_per_test": 0.926438, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 94.798, "latency": 986.856, "stderr": 2.149, "cost_per_test": 10.254175, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 94.599, "latency": 836.886, "stderr": 2.138, "cost_per_test": 4.339603, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 94.553, "latency": 318.119, "stderr": 2.621, "cost_per_test": 0.64171, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 94.52, "latency": 470.162, "stderr": 2.646, "cost_per_test": 0.708927, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 94.516, "latency": 1126.576, "stderr": 1.95, "cost_per_test": 12.38182, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 94.49, "latency": 3508.124, "stderr": 1.842, "cost_per_test": 1.530774, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 94.337, "latency": 1922.826, "stderr": 2.045, "cost_per_test": 2.642852, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-legal": { "accuracy": 94.319, "latency": 2681.332, "stderr": 1.839, "cost_per_test": 0.057691, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 93.826, "latency": 2414.0, "stderr": 2.285, "cost_per_test": 0.809076, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 93.821, "latency": 282.896, "stderr": 2.095, "cost_per_test": 1.899416, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 93.698, "latency": 3030.813, "stderr": 3.593, "cost_per_test": 22.907678, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 93.59, "latency": 2062.802, "stderr": 2.623, "cost_per_test": 9.360291, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 93.249, "latency": 1266.05, "stderr": 2.415, "cost_per_test": 2.204201, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 93.206, "latency": 3850.231, "stderr": 3.138, "cost_per_test": 8.132992, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 92.926, "latency": 3278.164, "stderr": 2.635, "cost_per_test": 0.238934, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 92.925, "latency": 303.781, "stderr": 2.219, "cost_per_test": 0.574895, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 92.819, "latency": 3359.798, "stderr": 2.602, "cost_per_test": 3.11111, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 92.435, "latency": 5897.488, "stderr": 2.313, "cost_per_test": 2.500509, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 92.161, "latency": 800.16, "stderr": 2.489, "cost_per_test": 0.394232, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 92.146, "latency": 1364.3, "stderr": 2.478, "cost_per_test": 2.657382, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 91.692, "latency": 1026.417, "stderr": 2.536, "cost_per_test": 3.793295, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 91.474, "latency": 274.578, "stderr": 1.976, "cost_per_test": 1.402227, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.2": { "accuracy": 91.257, "latency": 931.988, "stderr": 2.374, "cost_per_test": 0.931196, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 91.229, "latency": 999.255, "stderr": 2.203, "cost_per_test": 0.81017, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 91.114, "latency": 2156.169, "stderr": 3.007, "cost_per_test": 0.374147, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 90.816, "latency": 663.425, "stderr": 2.648, "cost_per_test": 3.159294, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 90.803, "latency": 1685.594, "stderr": 2.474, "cost_per_test": 1.561613, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 90.529, "latency": 245.802, "stderr": 2.623, "cost_per_test": 0.701623, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 90.496, "latency": 5711.873, "stderr": 4.683, "cost_per_test": 6.862578, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 90.283, "latency": 1047.406, "stderr": 2.565, "cost_per_test": 3.45972, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 90.249, "latency": 97.234, "stderr": 2.59, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "stepfun/step-5-preview": { "accuracy": 89.408, "latency": 2028.962, "stderr": 3.661, "cost_per_test": 0.905473, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 89.275, "latency": 2705.795, "stderr": 3.647, "cost_per_test": 1.314815, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 88.772, "latency": 7418.628, "stderr": 4.698, "cost_per_test": 5.193942, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "inception/mercury-3": { "accuracy": 88.551, "latency": 472.047, "stderr": 2.913, "cost_per_test": 0.076792, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "thinkingmachines/inkling": { "accuracy": 88.485, "latency": 910.936, "stderr": 2.857, "cost_per_test": 0.496174, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 88.212, "latency": 462.415, "stderr": 3.469, "cost_per_test": 4.040025, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 87.936, "latency": 298.421, "stderr": 2.581, "cost_per_test": 0.506452, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 87.857, "latency": 134.685, "stderr": 2.664, "cost_per_test": 0.416767, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 87.356, "latency": 781.127, "stderr": 4.19, "cost_per_test": 0.081431, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 87.176, "latency": 3297.206, "stderr": 3.353, "cost_per_test": 0.553927, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 86.824, "latency": 145.057, "stderr": 3.081, "cost_per_test": 0.136684, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 86.658, "latency": 848.765, "stderr": 3.085, "cost_per_test": 1.786099, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 86.614, "latency": 1384.623, "stderr": 2.951, "cost_per_test": 0.400672, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 85.452, "latency": 830.651, "stderr": 3.503, "cost_per_test": 0.505144, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 84.385, "latency": 998.171, "stderr": 3.091, "cost_per_test": 0.155212, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "tencent/hy4": { "accuracy": 84.258, "latency": 5571.399, "stderr": 6.239, "cost_per_test": 2.397715, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 83.436, "latency": 232.285, "stderr": 3.06, "cost_per_test": 1.804209, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 82.755, "latency": 322.342, "stderr": 3.924, "cost_per_test": 0.321248, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 81.669, "latency": 1608.673, "stderr": 3.886, "cost_per_test": 1.371176, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.3": { "accuracy": 81.212, "latency": 79.862, "stderr": 3.744, "cost_per_test": 0.233416, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 80.779, "latency": 387.77, "stderr": 4.363, "cost_per_test": 0.119406, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 80.726, "latency": 477.359, "stderr": 3.916, "cost_per_test": 0.028475, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 79.444, "latency": 696.803, "stderr": 4.502, "cost_per_test": 0.848896, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 75.743, "latency": 48.195, "stderr": 3.902, "cost_per_test": 0.13408, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 74.397, "latency": 1152.919, "stderr": 4.038, "cost_per_test": 0.18555, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 73.839, "latency": 71.497, "stderr": 3.946, "cost_per_test": 0.055137, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 73.772, "latency": 1671.994, "stderr": 4.948, "cost_per_test": 1.329631, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 73.53, "latency": 538.653, "stderr": 3.833, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 72.188, "latency": 334.244, "stderr": 4.417, "cost_per_test": 0.106495, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 71.555, "latency": 372.619, "stderr": 4.086, "cost_per_test": 0.056492, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 69.817, "latency": 232.734, "stderr": 4.727, "cost_per_test": 0.036118, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 69.597, "latency": 587.499, "stderr": 4.538, "cost_per_test": 0.21973, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "poolside/laguna-m.1": { "accuracy": 69.187, "latency": 474.816, "stderr": 3.476, "cost_per_test": 0.199185, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 67.923, "latency": 188.923, "stderr": 4.367, "cost_per_test": 0.290063, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 67.789, "latency": 618.768, "stderr": 4.381, "cost_per_test": 1.063116, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 62.795, "latency": 396.297, "stderr": 4.13, "cost_per_test": 0.09495, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 62.305, "latency": 229.682, "stderr": 4.367, "cost_per_test": 0.532775, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 57.555, "latency": 51.629, "stderr": 4.978, "cost_per_test": 0.048442, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 56.579, "latency": 19.508, "stderr": 4.937, "cost_per_test": 0.013992, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 45.376, "latency": 661.96, "stderr": 4.286, "cost_per_test": 0.005274, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "weighted_pass__immigration": { "anthropic/claude-opus-5-5": { "accuracy": 90.506, "latency": 7783.489, "stderr": 3.843, "cost_per_test": 28.846858, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 89.867, "latency": 1545.533, "stderr": 4.933, "cost_per_test": 4.782473, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 88.941, "latency": 4229.709, "stderr": 5.791, "cost_per_test": 29.748136, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-carbon": { "accuracy": 87.445, "latency": 2577.035, "stderr": 4.701, "cost_per_test": 7.345528, "temperature": null, "top_p": null, "max_output_tokens": 1048576, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 86.928, "latency": 2091.212, "stderr": 6.787, "cost_per_test": 8.705325, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 85.354, "latency": 2227.72, "stderr": 7.505, "cost_per_test": 2.139853, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 85.35, "latency": 2309.031, "stderr": 6.259, "cost_per_test": 5.522954, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 85.296, "latency": 1309.204, "stderr": 6.529, "cost_per_test": 5.041009, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 84.347, "latency": 2079.496, "stderr": 5.891, "cost_per_test": 14.716249, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 84.075, "latency": 372.58, "stderr": 7.185, "cost_per_test": 1.011872, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 83.773, "latency": 145.322, "stderr": 6.437, "cost_per_test": 0.576237, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-legal": { "accuracy": 83.657, "latency": 2188.906, "stderr": 7.29, "cost_per_test": 0.050144, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 83.475, "latency": 6121.239, "stderr": 8.164, "cost_per_test": 15.318633, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 83.421, "latency": 5257.651, "stderr": 5.767, "cost_per_test": 3.326554, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 83.175, "latency": 522.726, "stderr": 7.265, "cost_per_test": 3.815712, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 83.117, "latency": 5391.649, "stderr": 7.066, "cost_per_test": 2.04155, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 82.857, "latency": 231.193, "stderr": 6.336, "cost_per_test": 1.096453, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 82.55, "latency": 305.943, "stderr": 6.309, "cost_per_test": 0.448482, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.5": { "accuracy": 82.311, "latency": 771.306, "stderr": 7.186, "cost_per_test": 1.548677, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 82.114, "latency": 2512.615, "stderr": 6.489, "cost_per_test": 8.597148, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 82.025, "latency": 4767.048, "stderr": 6.62, "cost_per_test": 3.223316, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling-2-small": { "accuracy": 81.821, "latency": 733.81, "stderr": 6.163, "cost_per_test": 0.307991, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k3": { "accuracy": 81.495, "latency": 852.202, "stderr": 7.888, "cost_per_test": 4.505049, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 81.289, "latency": 1086.925, "stderr": 7.018, "cost_per_test": 0.562099, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse-wonder": { "accuracy": 81.2, "latency": 1055.89, "stderr": 7.615, "cost_per_test": 4.615585, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 81.155, "latency": 5565.396, "stderr": 7.633, "cost_per_test": 29.782421, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 80.986, "latency": 1290.179, "stderr": 7.784, "cost_per_test": 9.559148, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 80.706, "latency": 389.141, "stderr": 7.246, "cost_per_test": 0.560419, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 80.694, "latency": 5146.748, "stderr": 6.942, "cost_per_test": 9.402071, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 80.657, "latency": 2999.169, "stderr": 7.595, "cost_per_test": 1.324131, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "fireworks/ember-1": { "accuracy": 80.576, "latency": 2203.778, "stderr": 6.556, "cost_per_test": 2.644131, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 80.509, "latency": 377.453, "stderr": 6.956, "cost_per_test": 0.749015, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 80.332, "latency": 586.852, "stderr": 7.907, "cost_per_test": 0.921916, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 80.189, "latency": 219.971, "stderr": 6.859, "cost_per_test": 0.884226, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 80.097, "latency": 147.165, "stderr": 7.353, "cost_per_test": 0.405548, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 79.384, "latency": 3943.62, "stderr": 8.232, "cost_per_test": 3.006031, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 79.236, "latency": 3025.432, "stderr": 6.519, "cost_per_test": 5.242893, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 79.199, "latency": 1629.748, "stderr": 7.413, "cost_per_test": 2.754829, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 79.031, "latency": 2363.885, "stderr": 7.593, "cost_per_test": 0.899309, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 78.75, "latency": 598.99, "stderr": 8.816, "cost_per_test": 0.650754, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-luna": { "accuracy": 78.594, "latency": 3576.368, "stderr": 8.935, "cost_per_test": 0.63176, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 78.593, "latency": 882.27, "stderr": 8.748, "cost_per_test": 0.482243, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 78.572, "latency": 2966.232, "stderr": 7.434, "cost_per_test": 1.017586, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 78.447, "latency": 4168.205, "stderr": 8.598, "cost_per_test": 0.281372, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 78.35, "latency": 728.3, "stderr": 8.403, "cost_per_test": 3.441413, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 78.068, "latency": 1192.0, "stderr": 8.926, "cost_per_test": 1.003117, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 78.046, "latency": 2445.483, "stderr": 7.935, "cost_per_test": 3.066554, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 77.81, "latency": 1048.83, "stderr": 8.892, "cost_per_test": 2.703862, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 77.663, "latency": 376.011, "stderr": 8.837, "cost_per_test": 0.081151, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "stepfun/step-5-preview": { "accuracy": 77.592, "latency": 2795.76, "stderr": 8.491, "cost_per_test": 1.125625, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 77.311, "latency": 1059.33, "stderr": 9.055, "cost_per_test": 0.440884, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 77.0, "latency": 866.165, "stderr": 7.278, "cost_per_test": 0.178499, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 76.913, "latency": 3289.995, "stderr": 7.373, "cost_per_test": 0.523583, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "meta/muse-spark-open": { "accuracy": 76.745, "latency": 1537.666, "stderr": 8.465, "cost_per_test": 0.570169, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 76.548, "latency": 1793.408, "stderr": 8.224, "cost_per_test": 0.234002, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 76.442, "latency": 1260.914, "stderr": 8.243, "cost_per_test": 0.296663, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4": { "accuracy": 76.387, "latency": 6986.35, "stderr": 9.526, "cost_per_test": 2.97227, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-sol": { "accuracy": 76.217, "latency": 1914.142, "stderr": 7.996, "cost_per_test": 6.6645, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 76.087, "latency": 812.146, "stderr": 8.196, "cost_per_test": 0.150825, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 75.953, "latency": 176.78, "stderr": 7.7, "cost_per_test": 0.283317, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4-oss": { "accuracy": 75.908, "latency": 2067.692, "stderr": 9.428, "cost_per_test": 0.952254, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "inception/mercury-3": { "accuracy": 75.725, "latency": 449.603, "stderr": 7.536, "cost_per_test": 0.076601, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 75.593, "latency": 902.658, "stderr": 8.908, "cost_per_test": 0.700549, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 75.554, "latency": 1561.245, "stderr": 8.508, "cost_per_test": 1.49714, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 75.048, "latency": 4515.312, "stderr": 8.147, "cost_per_test": 1.089, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 74.597, "latency": 659.266, "stderr": 8.528, "cost_per_test": 0.026153, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 74.549, "latency": 1272.83, "stderr": 7.195, "cost_per_test": 0.124704, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 74.064, "latency": 851.042, "stderr": 8.604, "cost_per_test": 1.91176, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 73.84, "latency": 71.817, "stderr": 8.127, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 73.479, "latency": 848.642, "stderr": 9.162, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 73.271, "latency": 705.891, "stderr": 9.271, "cost_per_test": 1.495632, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 72.894, "latency": 185.396, "stderr": 9.888, "cost_per_test": 0.389491, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 71.531, "latency": 128.854, "stderr": 8.032, "cost_per_test": 0.088481, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 69.874, "latency": 356.501, "stderr": 8.499, "cost_per_test": 0.229174, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-6.1-terra": { "accuracy": 69.793, "latency": 5272.469, "stderr": 9.655, "cost_per_test": 3.88229, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 68.693, "latency": 60.521, "stderr": 9.72, "cost_per_test": 0.102995, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 68.382, "latency": 2027.42, "stderr": 8.841, "cost_per_test": 1.631269, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "baseten/nemotron-3.5-super-vl-120b-a12b": { "accuracy": 68.213, "latency": 69.703, "stderr": 7.787, "cost_per_test": 0.06269, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Baseten", "harness": null }, "grok/grok-4.3": { "accuracy": 67.726, "latency": 242.659, "stderr": 8.228, "cost_per_test": 0.366775, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 66.867, "latency": 823.783, "stderr": 10.101, "cost_per_test": 0.603054, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 65.25, "latency": 559.812, "stderr": 7.857, "cost_per_test": 0.159001, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 64.376, "latency": 38.568, "stderr": 7.908, "cost_per_test": 0.046126, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-m.1": { "accuracy": 64.305, "latency": 610.78, "stderr": 8.96, "cost_per_test": 0.296042, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 61.671, "latency": 201.446, "stderr": 9.534, "cost_per_test": 0.033288, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 60.533, "latency": 255.303, "stderr": 9.866, "cost_per_test": 0.300956, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 57.38, "latency": 473.789, "stderr": 8.96, "cost_per_test": 0.036168, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 57.024, "latency": 21.669, "stderr": 9.462, "cost_per_test": 0.014207, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 53.543, "latency": 7303.148, "stderr": 13.701, "cost_per_test": 4.997493, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 51.559, "latency": 394.752, "stderr": 9.455, "cost_per_test": 0.099916, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 50.44, "latency": 173.024, "stderr": 9.552, "cost_per_test": 0.003083, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k3.1-pro": { "accuracy": 32.709, "latency": 7116.076, "stderr": 12.681, "cost_per_test": 8.488087, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null } } } }