{ "metadata": { "benchmark": "Harvey's Legal Agent Benchmark", "slug": "hlab", "description": "Tests an agent's ability to complete legal work using documents, spreadsheets, presentations, and file-system tools", "benchmark_id": "legal_agent_benchmark", "family": "legal_agent_benchmark", "version": "1", "updated": "2026-10-07", "dataset_type": "public", "industry": "legal", "tasks": { "overall": "Overall", "antitrust_competition": "Antitrust Competition", "arbitration_international_dispute_resolution": "Arbitration International Dispute Resolution", "banking_finance": "Banking Finance", "bankruptcy_restructuring": "Bankruptcy Restructuring", "capital_markets": "Capital Markets", "corporate_governance": "Corporate Governance", "corporate_ma": "Corporate Ma", "data_privacy_cybersecurity": "Data Privacy Cybersecurity", "emerging_companies_venture_capital": "Emerging Companies Venture Capital", "employment_labor": "Employment Labor", "energy_natural_resources": "Energy Natural Resources", "environmental_esg": "Environmental Esg", "funds_asset_management": "Funds Asset Management", "healthcare_life_sciences": "Healthcare Life Sciences", "immigration": "Immigration", "insurance": "Insurance", "intellectual_property": "Intellectual Property", "international_trade_sanctions": "International Trade Sanctions", "litigation_dispute_resolution": "Litigation Dispute Resolution", "real_estate": "Real Estate", "structured_finance_securitization": "Structured Finance Securitization", "tax": "Tax", "trusts_estates_private_client": "Trusts Estates Private Client", "white_collar_defense_investigations": "White Collar Defense Investigations", "criteria_pass_rate": "Criteria Pass Rate", "criteria_pass_rate__antitrust_competition": "Antitrust Competition", "criteria_pass_rate__arbitration_international_dispute_resolution": "Arbitration International Dispute Resolution", "criteria_pass_rate__banking_finance": "Banking Finance", "criteria_pass_rate__bankruptcy_restructuring": "Bankruptcy Restructuring", "criteria_pass_rate__capital_markets": "Capital Markets", "criteria_pass_rate__corporate_governance": "Corporate Governance", "criteria_pass_rate__corporate_ma": "Corporate Ma", "criteria_pass_rate__data_privacy_cybersecurity": "Data Privacy Cybersecurity", "criteria_pass_rate__emerging_companies_venture_capital": "Emerging Companies Venture Capital", "criteria_pass_rate__employment_labor": "Employment Labor", "criteria_pass_rate__energy_natural_resources": "Energy Natural Resources", "criteria_pass_rate__environmental_esg": "Environmental Esg", "criteria_pass_rate__funds_asset_management": "Funds Asset Management", "criteria_pass_rate__healthcare_life_sciences": "Healthcare Life Sciences", "criteria_pass_rate__immigration": "Immigration", "criteria_pass_rate__insurance": "Insurance", "criteria_pass_rate__intellectual_property": "Intellectual Property", "criteria_pass_rate__international_trade_sanctions": "International Trade Sanctions", "criteria_pass_rate__litigation_dispute_resolution": "Litigation Dispute Resolution", "criteria_pass_rate__real_estate": "Real Estate", "criteria_pass_rate__structured_finance_securitization": "Structured Finance Securitization", "criteria_pass_rate__tax": "Tax", "criteria_pass_rate__trusts_estates_private_client": "Trusts Estates Private Client", "criteria_pass_rate__white_collar_defense_investigations": "White Collar Defense Investigations" }, "models": [ "alibaba/qwen3.6-plus", "alibaba/qwen3.7-max", "alibaba/qwen3.7-plus", "alibaba/qwen3.8-27b", "alibaba/qwen3.8-max", "ant/ling-3.0-flash-2607", "ant/ling-3.0-flash-af-rc3", "anthropic/claude-fable-5", "anthropic/claude-fable-5-1", "anthropic/claude-haiku-4-5-20251001-thinking", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-4-7", "anthropic/claude-opus-4-8", "anthropic/claude-opus-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash", "fireworks/ember-1", "fireworks/nemotron-lightning-3p5-30b-a3b", "google/gemini-3-flash-preview", "google/gemini-3.1-flash-lite-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", "google/gemini-3.6-flash", "google/gemini-3.7-flash", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.20-0309-reasoning", "grok/grok-4.3", "grok/grok-4.5", "grok/grok-4.6", "grok/grok-4.7", "inception/mercury-2.5", "kimi/kimi-k2.5-thinking", "kimi/kimi-k2.6", "kimi/kimi-k3", "meta/muse_spark_1_1", "meta/muse_spark_1_2", "meta/muse_spark_1_3", "meta/muse_spark_1_3_max", "minimax/MiniMax-M2.7", "minimax/MiniMax-M3", "mistralai/mistral-large-4", "mistralai/mistral-medium-3.5", "nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-5.4-2026-03-05", "openai/gpt-5.4-mini-2026-03-17", "openai/gpt-5.4-nano-2026-03-17", "openai/gpt-5.5", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/gpt-5.6-terra", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6-sol", "openai/gpt-6.1-sol", "poolside/laguna-m.1", "poolside/laguna-xs.2", "stepfun/step-5-preview", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.5", "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.1", "zai/glm-5.2", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": true, "total_models": 76 }, "tasks": { "overall": { "meta/muse_spark_1_2": { "accuracy": 25.417, "latency": 1456.298, "stderr": 3.667, "cost_per_test": 2.08555, "token_totals": { "input_tokens": 608754402, "output_tokens": 10507621, "reasoning_tokens": 826686, "cache_read_tokens": 512433248, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 23.75, "latency": 882.844, "stderr": 3.547, "cost_per_test": 2.25729, "token_totals": { "input_tokens": 1051561571, "output_tokens": 12477171, "reasoning_tokens": 2210430, "cache_read_tokens": 1009226293, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 22.917, "latency": 1024.261, "stderr": 3.416, "cost_per_test": 2.75654, "token_totals": { "input_tokens": 835346220, "output_tokens": 8208587, "reasoning_tokens": 1656318, "cache_read_tokens": 690282338, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 20.0, "latency": 740.833, "stderr": 2.943, "cost_per_test": 0.800257, "token_totals": { "input_tokens": 283802188, "output_tokens": 5292532, "reasoning_tokens": 497732, "cache_read_tokens": 263652672, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 19.583, "latency": 2634.915, "stderr": 3.31, "cost_per_test": 10.54836, "token_totals": { "input_tokens": 826582568, "output_tokens": 18828918, "reasoning_tokens": 5044847, "cache_read_tokens": 636080391, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 15.833, "latency": 2364.651, "stderr": 2.958, "cost_per_test": 4.181392, "token_totals": { "input_tokens": 826343789, "output_tokens": 12798926, "reasoning_tokens": null, "cache_read_tokens": 553737728, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.6": { "accuracy": 15.833, "latency": 2705.048, "stderr": 2.645, "cost_per_test": 4.008154, "token_totals": { "input_tokens": 655024488, "output_tokens": 16284504, "reasoning_tokens": 3704862, "cache_read_tokens": 617851648, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.5": { "accuracy": 12.917, "latency": 614.705, "stderr": 2.534, "cost_per_test": 2.01955, "token_totals": { "input_tokens": 220580311, "output_tokens": 9143344, "reasoning_tokens": 326470, "cache_read_tokens": 198038400, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 12.917, "latency": 837.646, "stderr": 2.679, "cost_per_test": 3.829773, "token_totals": { "input_tokens": 797226748, "output_tokens": 9686458, "reasoning_tokens": 3624789, "cache_read_tokens": 769409022, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.7": { "accuracy": 12.5, "latency": 2525.239, "stderr": 2.663, "cost_per_test": 11.130983, "token_totals": { "input_tokens": 1000748161, "output_tokens": 20963877, "reasoning_tokens": 8944068, "cache_read_tokens": 911133568, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 11.25, "latency": 1031.581, "stderr": 2.543, "cost_per_test": 0.085646, "token_totals": { "input_tokens": 751303984, "output_tokens": 11580281, "reasoning_tokens": 5709544, "cache_read_tokens": 715360896, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 11.25, "latency": 1118.634, "stderr": 2.415, "cost_per_test": 4.033621, "token_totals": { "input_tokens": 892576388, "output_tokens": 12582101, "reasoning_tokens": 5612426, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 11.25, "latency": 1613.035, "stderr": 2.148, "cost_per_test": 19.225253, "token_totals": { "input_tokens": 939267211, "output_tokens": 13345157, "reasoning_tokens": null, "cache_read_tokens": 880023896, "cache_write_tokens": 59236069 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 10.833, "latency": 1553.048, "stderr": 2.453, "cost_per_test": 0.216614, "token_totals": { "input_tokens": 632411237, "output_tokens": 11283302, "reasoning_tokens": 5261809, "cache_read_tokens": 600189312, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 10.417, "latency": 2952.433, "stderr": 2.287, "cost_per_test": 2.365468, "token_totals": { "input_tokens": 702241099, "output_tokens": 9014364, "reasoning_tokens": 3675937, "cache_read_tokens": 671264128, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 10.0, "latency": 1760.871, "stderr": 2.415, "cost_per_test": 3.656207, "token_totals": { "input_tokens": 1171742866, "output_tokens": 16120993, "reasoning_tokens": 5514918, "cache_read_tokens": 1066501383, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 9.583, "latency": 1447.794, "stderr": 2.148, "cost_per_test": 10.222944, "token_totals": { "input_tokens": 1143265902, "output_tokens": 13823660, "reasoning_tokens": null, "cache_read_tokens": 1089433052, "cache_write_tokens": 53824786 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 9.167, "latency": 1741.944, "stderr": 2.439, "cost_per_test": 0.997491, "token_totals": { "input_tokens": 1090091574, "output_tokens": 12155839, "reasoning_tokens": 0, "cache_read_tokens": 1056014080, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "tencent/hy4-preview": { "accuracy": 9.167, "latency": 1764.664, "stderr": 1.998, "cost_per_test": 0.98378, "token_totals": { "input_tokens": 1072794048, "output_tokens": 13132219, "reasoning_tokens": 4467767, "cache_read_tokens": 1022096320, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "fireworks/ember-1": { "accuracy": 9.167, "latency": 2665.557, "stderr": 2.439, "cost_per_test": 3.333599, "token_totals": { "input_tokens": 649364400, "output_tokens": 7643857, "reasoning_tokens": 2533467, "cache_read_tokens": 615821913, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 8.75, "latency": 452.101, "stderr": 2.148, "cost_per_test": 2.556743, "token_totals": { "input_tokens": 811642014, "output_tokens": 14269397, "reasoning_tokens": 3601488, "cache_read_tokens": 753832806, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 8.333, "latency": 1014.028, "stderr": 2.148, "cost_per_test": 0.059087, "token_totals": { "input_tokens": 744084543, "output_tokens": 7335131, "reasoning_tokens": 2960038, "cache_read_tokens": 723421312, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 8.333, "latency": 2558.175, "stderr": 1.998, "cost_per_test": 4.302903, "token_totals": { "input_tokens": 1456140757, "output_tokens": 16324375, "reasoning_tokens": null, "cache_read_tokens": 1398312225, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 7.5, "latency": 1313.321, "stderr": 2.148, "cost_per_test": 0.166206, "token_totals": { "input_tokens": 805739429, "output_tokens": 9454727, "reasoning_tokens": 4381477, "cache_read_tokens": 785343488, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 7.083, "latency": 1250.956, "stderr": 1.998, "cost_per_test": 2.057915, "token_totals": { "input_tokens": 582413825, "output_tokens": 8379723, "reasoning_tokens": 3962778, "cache_read_tokens": 530965248, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 6.667, "latency": 579.44, "stderr": 1.832, "cost_per_test": 0.173471, "token_totals": { "input_tokens": 721200533, "output_tokens": 8383826, "reasoning_tokens": 4048611, "cache_read_tokens": 699334016, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 6.667, "latency": 1483.696, "stderr": 1.998, "cost_per_test": 10.80884, "token_totals": { "input_tokens": 1150494190, "output_tokens": 13074265, "reasoning_tokens": 1317583, "cache_read_tokens": 1081804814, "cache_write_tokens": 68683898 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 6.667, "latency": 3136.646, "stderr": 1.646, "cost_per_test": 0.570286, "token_totals": { "input_tokens": 3914223047, "output_tokens": 18369668, "reasoning_tokens": null, "cache_read_tokens": 3828746240, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 6.667, "latency": 3338.269, "stderr": 1.646, "cost_per_test": 23.669537, "token_totals": { "input_tokens": 2757683085, "output_tokens": 27559556, "reasoning_tokens": 6673656, "cache_read_tokens": 2623329853, "cache_write_tokens": 134339560 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 6.667, "latency": 6749.446, "stderr": 1.174, "cost_per_test": 46.211522, "token_totals": { "input_tokens": 2140493271, "output_tokens": 25042909, "reasoning_tokens": 9339413, "cache_read_tokens": 1833706187, "cache_write_tokens": 306775910 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 5.417, "latency": 1489.086, "stderr": 1.174, "cost_per_test": 26.155899, "token_totals": { "input_tokens": 1119192487, "output_tokens": 10562148, "reasoning_tokens": 3650237, "cache_read_tokens": 1049116892, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 5.417, "latency": 2594.18, "stderr": 1.755, "cost_per_test": 3.933528, "token_totals": { "input_tokens": 1310994009, "output_tokens": 13514557, "reasoning_tokens": 5869694, "cache_read_tokens": 1274733716, "cache_write_tokens": 36236558 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 5.0, "latency": 1122.707, "stderr": 1.646, "cost_per_test": 3.042346, "token_totals": { "input_tokens": 387928425, "output_tokens": 6856417, "reasoning_tokens": null, "cache_read_tokens": 345650547, "cache_write_tokens": 42275246 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 5.0, "latency": 2331.577, "stderr": 1.646, "cost_per_test": 8.953859, "token_totals": { "input_tokens": 3340913248, "output_tokens": 20212831, "reasoning_tokens": null, "cache_read_tokens": 3252147720, "cache_write_tokens": 88748346 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 4.167, "latency": 1351.134, "stderr": 1.646, "cost_per_test": 1.460248, "token_totals": { "input_tokens": 1114902164, "output_tokens": 8397929, "reasoning_tokens": null, "cache_read_tokens": 1070555232, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 3.75, "latency": 401.522, "stderr": 1.431, "cost_per_test": 0.737465, "token_totals": { "input_tokens": 321058309, "output_tokens": 4167657, "reasoning_tokens": 272693, "cache_read_tokens": 303855232, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 3.75, "latency": 733.847, "stderr": 1.174, "cost_per_test": 4.597894, "token_totals": { "input_tokens": 372470134, "output_tokens": 4184434, "reasoning_tokens": 1267952, "cache_read_tokens": 341294080, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 3.75, "latency": 3175.128, "stderr": 1.469, "cost_per_test": 21.37949, "token_totals": { "input_tokens": 2462576678, "output_tokens": 38472452, "reasoning_tokens": 24925173, "cache_read_tokens": 2286257168, "cache_write_tokens": 176308820 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 3.333, "latency": 984.49, "stderr": 1.174, "cost_per_test": 1.895602, "token_totals": { "input_tokens": 434069662, "output_tokens": 9410323, "reasoning_tokens": 2894511, "cache_read_tokens": 366081207, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 2.917, "latency": 1051.83, "stderr": 1.362, "cost_per_test": 0.303552, "token_totals": { "input_tokens": 1460813994, "output_tokens": 12095727, "reasoning_tokens": 6619071, "cache_read_tokens": 1422964599, "cache_write_tokens": 37829190 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 2.917, "latency": 3464.426, "stderr": 1.362, "cost_per_test": 16.612583, "token_totals": { "input_tokens": 4750998856, "output_tokens": 53560846, "reasoning_tokens": 35373065, "cache_read_tokens": 4582495471, "cache_write_tokens": 168486127 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 2.5, "latency": 478.414, "stderr": 0.833, "cost_per_test": 2.116049, "token_totals": { "input_tokens": 606812300, "output_tokens": 8678382, "reasoning_tokens": 3806922, "cache_read_tokens": 543998500, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 2.5, "latency": 1370.296, "stderr": 0.833, "cost_per_test": 10.374554, "token_totals": { "input_tokens": 1221154582, "output_tokens": 10565145, "reasoning_tokens": 2786144, "cache_read_tokens": 1150617952, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 2.083, "latency": 544.226, "stderr": 0.833, "cost_per_test": 0.107095, "token_totals": { "input_tokens": 224714337, "output_tokens": 3198839, "reasoning_tokens": 0, "cache_read_tokens": 203250560, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling": { "accuracy": 2.083, "latency": 1103.692, "stderr": 0.833, "cost_per_test": 1.252023, "token_totals": { "input_tokens": 575866616, "output_tokens": 7829496, "reasoning_tokens": null, "cache_read_tokens": 552512512, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 1.667, "latency": 419.277, "stderr": 0.0, "cost_per_test": 0.036252, "token_totals": { "input_tokens": 252892290, "output_tokens": 3100583, "reasoning_tokens": 145588, "cache_read_tokens": 232673984, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-sol": { "accuracy": 1.667, "latency": 820.097, "stderr": 0.823, "cost_per_test": 3.358015, "token_totals": { "input_tokens": 823114531, "output_tokens": 7639311, "reasoning_tokens": 3222223, "cache_read_tokens": 791317282, "cache_write_tokens": 31785134 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 1.667, "latency": 1405.253, "stderr": 0.833, "cost_per_test": 0.818741, "token_totals": { "input_tokens": 373247916, "output_tokens": 3422680, "reasoning_tokens": 0, "cache_read_tokens": 341806731, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 1.667, "latency": 1870.966, "stderr": 0.833, "cost_per_test": 0.174542, "token_totals": { "input_tokens": 201049757, "output_tokens": 5401207, "reasoning_tokens": null, "cache_read_tokens": 191047296, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 1.667, "latency": 1923.25, "stderr": 0.833, "cost_per_test": 1.57592, "token_totals": { "input_tokens": 249242887, "output_tokens": 2999380, "reasoning_tokens": 202288, "cache_read_tokens": 203585792, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 1.25, "latency": 161.487, "stderr": 0.833, "cost_per_test": 0.046313, "token_totals": { "input_tokens": 146486877, "output_tokens": 4726655, "reasoning_tokens": null, "cache_read_tokens": 110128578, "cache_write_tokens": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 1.25, "latency": 656.357, "stderr": 0.833, "cost_per_test": 0.383981, "token_totals": { "input_tokens": 993049543, "output_tokens": 9388666, "reasoning_tokens": 2979345, "cache_read_tokens": 909992290, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 1.25, "latency": 888.433, "stderr": 0.833, "cost_per_test": 3.940956, "token_totals": { "input_tokens": 488567465, "output_tokens": 3914542, "reasoning_tokens": 154232, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 1.25, "latency": 2113.923, "stderr": 0.716, "cost_per_test": 3.394654, "token_totals": { "input_tokens": 4127341953, "output_tokens": 54990908, "reasoning_tokens": 37472036, "cache_read_tokens": 4002588737, "cache_write_tokens": 124738510 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.833, "latency": 361.116, "stderr": 0.833, "cost_per_test": 0.416609, "token_totals": { "input_tokens": 129616489, "output_tokens": 3945656, "reasoning_tokens": 242225, "cache_read_tokens": 114566187, "cache_write_tokens": 15031667 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.833, "latency": 1024.886, "stderr": 0.833, "cost_per_test": 3.202399, "token_totals": { "input_tokens": 802970365, "output_tokens": 9301238, "reasoning_tokens": 2973597, "cache_read_tokens": 740704262, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 0.417, "latency": 168.721, "stderr": 0.0, "cost_per_test": 0.389144, "token_totals": { "input_tokens": 88472856, "output_tokens": 3683568, "reasoning_tokens": 464382, "cache_read_tokens": 71653723, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.417, "latency": 2048.539, "stderr": 0.0, "cost_per_test": null, "token_totals": { "input_tokens": 305706389, "output_tokens": 3758581, "reasoning_tokens": null, "cache_read_tokens": 48583680, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.417, "latency": 3694.945, "stderr": 0.0, "cost_per_test": 6.074265, "token_totals": { "input_tokens": 439770514, "output_tokens": 5184631, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 60.554, "stderr": 0.0, "cost_per_test": 0.084048, "token_totals": { "input_tokens": 160066368, "output_tokens": 769494, "reasoning_tokens": 360545, "cache_read_tokens": 138155856, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 131.54, "stderr": 0.0, "cost_per_test": 0.192614, "token_totals": { "input_tokens": 159575776, "output_tokens": 1621389, "reasoning_tokens": 1524161, "cache_read_tokens": 55653173, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 202.513, "stderr": 0.0, "cost_per_test": 0.264375, "token_totals": { "input_tokens": 225950385, "output_tokens": 1424392, "reasoning_tokens": 580723, "cache_read_tokens": 190051973, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 232.895, "stderr": 0.0, "cost_per_test": 0.331805, "token_totals": { "input_tokens": 234750879, "output_tokens": 3541635, "reasoning_tokens": 1058834, "cache_read_tokens": 146158528, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 358.218, "stderr": 0.0, "cost_per_test": 0.045991, "token_totals": { "input_tokens": 212025841, "output_tokens": 4296441, "reasoning_tokens": null, "cache_read_tokens": 166167424, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 363.008, "stderr": 0.0, "cost_per_test": 0.184287, "token_totals": { "input_tokens": 315810901, "output_tokens": 2855013, "reasoning_tokens": 1338430, "cache_read_tokens": 261179392, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 373.931, "stderr": 0.0, "cost_per_test": 0.489594, "token_totals": { "input_tokens": 194662499, "output_tokens": 2889064, "reasoning_tokens": 177130, "cache_read_tokens": 11540069, "cache_write_tokens": 18939 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 436.69, "stderr": 0.0, "cost_per_test": 1.149757, "token_totals": { "input_tokens": 230732445, "output_tokens": 1350000, "reasoning_tokens": 503310, "cache_read_tokens": 204622317, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 465.122, "stderr": 0.0, "cost_per_test": 0.527685, "token_totals": { "input_tokens": 547314889, "output_tokens": 1917329, "reasoning_tokens": 177714, "cache_read_tokens": 495461278, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 498.074, "stderr": 0.0, "cost_per_test": 0.555636, "token_totals": { "input_tokens": 218148148, "output_tokens": 2889617, "reasoning_tokens": 207437, "cache_read_tokens": 200898240, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 502.903, "stderr": 0.0, "cost_per_test": 0.986814, "token_totals": { "input_tokens": 248182901, "output_tokens": 1198831, "reasoning_tokens": 239427, "cache_read_tokens": 216554703, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 529.178, "stderr": 0.0, "cost_per_test": 0.234226, "token_totals": { "input_tokens": 214653131, "output_tokens": 1748791, "reasoning_tokens": 535291, "cache_read_tokens": 170857216, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 681.27, "stderr": 0.0, "cost_per_test": 1.001493, "token_totals": { "input_tokens": 440584578, "output_tokens": 6595483, "reasoning_tokens": 5276888, "cache_read_tokens": 389590016, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 876.217, "stderr": 0.0, "cost_per_test": 3.598304, "token_totals": { "input_tokens": 665491305, "output_tokens": 5254641, "reasoning_tokens": 3014803, "cache_read_tokens": 623007616, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 1194.199, "stderr": 0.0, "cost_per_test": 0.284127, "token_totals": { "input_tokens": 322083675, "output_tokens": 1329814, "reasoning_tokens": 131024, "cache_read_tokens": 189201152, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1508.275, "stderr": 0.0, "cost_per_test": 0.351359, "token_totals": { "input_tokens": 236174566, "output_tokens": 2527211, "reasoning_tokens": 124700, "cache_read_tokens": 219868250, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 1736.203, "stderr": 0.0, "cost_per_test": 0.028287, "token_totals": { "input_tokens": 216862233, "output_tokens": 2963970, "reasoning_tokens": null, "cache_read_tokens": 201035952, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "criteria_pass_rate": { "meta/muse_spark_1_3_max": { "accuracy": 94.742, "latency": 882.844, "stderr": 0.275, "cost_per_test": 2.25729, "token_totals": { "input_tokens": 1051561571, "output_tokens": 12477171, "reasoning_tokens": 2210430, "cache_read_tokens": 1009226293, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 94.522, "latency": 1456.298, "stderr": 0.278, "cost_per_test": 2.08555, "token_totals": { "input_tokens": 608754402, "output_tokens": 10507621, "reasoning_tokens": 826686, "cache_read_tokens": 512433248, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 94.037, "latency": 2634.915, "stderr": 0.285, "cost_per_test": 10.54836, "token_totals": { "input_tokens": 826582568, "output_tokens": 18828918, "reasoning_tokens": 5044847, "cache_read_tokens": 636080391, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 92.859, "latency": 740.833, "stderr": 0.324, "cost_per_test": 0.800257, "token_totals": { "input_tokens": 283802188, "output_tokens": 5292532, "reasoning_tokens": 497732, "cache_read_tokens": 263652672, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 92.517, "latency": 2705.048, "stderr": 0.317, "cost_per_test": 4.008154, "token_totals": { "input_tokens": 655024488, "output_tokens": 16284504, "reasoning_tokens": 3704862, "cache_read_tokens": 617851648, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 92.209, "latency": 1024.261, "stderr": 0.327, "cost_per_test": 2.75654, "token_totals": { "input_tokens": 835346220, "output_tokens": 8208587, "reasoning_tokens": 1656318, "cache_read_tokens": 690282338, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 91.084, "latency": 837.646, "stderr": 0.343, "cost_per_test": 3.829773, "token_totals": { "input_tokens": 797226748, "output_tokens": 9686458, "reasoning_tokens": 3624789, "cache_read_tokens": 769409022, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 90.693, "latency": 1553.048, "stderr": 0.351, "cost_per_test": 0.216614, "token_totals": { "input_tokens": 632411237, "output_tokens": 11283302, "reasoning_tokens": 5261809, "cache_read_tokens": 600189312, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.5": { "accuracy": 90.549, "latency": 614.705, "stderr": 0.352, "cost_per_test": 2.01955, "token_totals": { "input_tokens": 220580311, "output_tokens": 9143344, "reasoning_tokens": 326470, "cache_read_tokens": 198038400, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.7": { "accuracy": 90.491, "latency": 2525.239, "stderr": 0.353, "cost_per_test": 11.130983, "token_totals": { "input_tokens": 1000748161, "output_tokens": 20963877, "reasoning_tokens": 8944068, "cache_read_tokens": 911133568, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 90.48, "latency": 1613.035, "stderr": 0.354, "cost_per_test": 19.225253, "token_totals": { "input_tokens": 939267211, "output_tokens": 13345157, "reasoning_tokens": null, "cache_read_tokens": 880023896, "cache_write_tokens": 59236069 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 90.23, "latency": 2665.557, "stderr": 0.357, "cost_per_test": 3.333599, "token_totals": { "input_tokens": 649364400, "output_tokens": 7643857, "reasoning_tokens": 2533467, "cache_read_tokens": 615821913, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 90.187, "latency": 1760.871, "stderr": 0.358, "cost_per_test": 3.656207, "token_totals": { "input_tokens": 1171742866, "output_tokens": 16120993, "reasoning_tokens": 5514918, "cache_read_tokens": 1066501383, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 90.013, "latency": 2952.433, "stderr": 0.361, "cost_per_test": 2.365468, "token_totals": { "input_tokens": 702241099, "output_tokens": 9014364, "reasoning_tokens": 3675937, "cache_read_tokens": 671264128, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 89.941, "latency": 1031.581, "stderr": 0.362, "cost_per_test": 0.085646, "token_totals": { "input_tokens": 751303984, "output_tokens": 11580281, "reasoning_tokens": 5709544, "cache_read_tokens": 715360896, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 89.666, "latency": 2558.175, "stderr": 0.366, "cost_per_test": 4.302903, "token_totals": { "input_tokens": 1456140757, "output_tokens": 16324375, "reasoning_tokens": null, "cache_read_tokens": 1398312225, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 89.405, "latency": 1764.664, "stderr": 0.37, "cost_per_test": 0.98378, "token_totals": { "input_tokens": 1072794048, "output_tokens": 13132219, "reasoning_tokens": 4467767, "cache_read_tokens": 1022096320, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 89.347, "latency": 2364.651, "stderr": 0.371, "cost_per_test": 4.181392, "token_totals": { "input_tokens": 826343789, "output_tokens": 12798926, "reasoning_tokens": null, "cache_read_tokens": 553737728, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 89.077, "latency": 1741.944, "stderr": 0.377, "cost_per_test": 0.997491, "token_totals": { "input_tokens": 1090091574, "output_tokens": 12155839, "reasoning_tokens": 0, "cache_read_tokens": 1056014080, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 88.815, "latency": 1014.028, "stderr": 0.382, "cost_per_test": 0.059087, "token_totals": { "input_tokens": 744084543, "output_tokens": 7335131, "reasoning_tokens": 2960038, "cache_read_tokens": 723421312, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 88.783, "latency": 1118.634, "stderr": 0.38, "cost_per_test": 4.033621, "token_totals": { "input_tokens": 892576388, "output_tokens": 12582101, "reasoning_tokens": 5612426, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 88.595, "latency": 452.101, "stderr": 0.382, "cost_per_test": 2.556743, "token_totals": { "input_tokens": 811642014, "output_tokens": 14269397, "reasoning_tokens": 3601488, "cache_read_tokens": 753832806, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 88.059, "latency": 1313.321, "stderr": 0.39, "cost_per_test": 0.166206, "token_totals": { "input_tokens": 805739429, "output_tokens": 9454727, "reasoning_tokens": 4381477, "cache_read_tokens": 785343488, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 87.856, "latency": 1447.794, "stderr": 0.393, "cost_per_test": 10.222944, "token_totals": { "input_tokens": 1143265902, "output_tokens": 13823660, "reasoning_tokens": null, "cache_read_tokens": 1089433052, "cache_write_tokens": 53824786 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 87.784, "latency": 2594.18, "stderr": 0.394, "cost_per_test": 3.933528, "token_totals": { "input_tokens": 1310994009, "output_tokens": 13514557, "reasoning_tokens": 5869694, "cache_read_tokens": 1274733716, "cache_write_tokens": 36236558 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 87.755, "latency": 1483.696, "stderr": 0.394, "cost_per_test": 10.80884, "token_totals": { "input_tokens": 1150494190, "output_tokens": 13074265, "reasoning_tokens": 1317583, "cache_read_tokens": 1081804814, "cache_write_tokens": 68683898 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 87.697, "latency": 3338.269, "stderr": 0.395, "cost_per_test": 23.669537, "token_totals": { "input_tokens": 2757683085, "output_tokens": 27559556, "reasoning_tokens": 6673656, "cache_read_tokens": 2623329853, "cache_write_tokens": 134339560 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 87.495, "latency": 579.44, "stderr": 0.398, "cost_per_test": 0.173471, "token_totals": { "input_tokens": 721200533, "output_tokens": 8383826, "reasoning_tokens": 4048611, "cache_read_tokens": 699334016, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 87.48, "latency": 1489.086, "stderr": 0.398, "cost_per_test": 26.155899, "token_totals": { "input_tokens": 1119192487, "output_tokens": 10562148, "reasoning_tokens": 3650237, "cache_read_tokens": 1049116892, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 86.655, "latency": 1122.707, "stderr": 0.409, "cost_per_test": 3.042346, "token_totals": { "input_tokens": 387928425, "output_tokens": 6856417, "reasoning_tokens": null, "cache_read_tokens": 345650547, "cache_write_tokens": 42275246 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 86.568, "latency": 6749.446, "stderr": 0.41, "cost_per_test": 46.211522, "token_totals": { "input_tokens": 2140493271, "output_tokens": 25042909, "reasoning_tokens": 9339413, "cache_read_tokens": 1833706187, "cache_write_tokens": 306775910 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 86.279, "latency": 1351.134, "stderr": 0.414, "cost_per_test": 1.460248, "token_totals": { "input_tokens": 1114902164, "output_tokens": 8397929, "reasoning_tokens": null, "cache_read_tokens": 1070555232, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 86.279, "latency": 2331.577, "stderr": 0.414, "cost_per_test": 8.953859, "token_totals": { "input_tokens": 3340913248, "output_tokens": 20212831, "reasoning_tokens": null, "cache_read_tokens": 3252147720, "cache_write_tokens": 88748346 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 85.598, "latency": 1250.956, "stderr": 0.422, "cost_per_test": 2.057915, "token_totals": { "input_tokens": 582413825, "output_tokens": 8379723, "reasoning_tokens": 3962778, "cache_read_tokens": 530965248, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 84.397, "latency": 401.522, "stderr": 0.437, "cost_per_test": 0.737465, "token_totals": { "input_tokens": 321058309, "output_tokens": 4167657, "reasoning_tokens": 272693, "cache_read_tokens": 303855232, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 84.368, "latency": 984.49, "stderr": 0.437, "cost_per_test": 1.895602, "token_totals": { "input_tokens": 434069662, "output_tokens": 9410323, "reasoning_tokens": 2894511, "cache_read_tokens": 366081207, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 83.514, "latency": 820.097, "stderr": 0.446, "cost_per_test": 3.358015, "token_totals": { "input_tokens": 823114531, "output_tokens": 7639311, "reasoning_tokens": 3222223, "cache_read_tokens": 791317282, "cache_write_tokens": 31785134 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 83.467, "latency": 1923.25, "stderr": 0.449, "cost_per_test": 1.57592, "token_totals": { "input_tokens": 249242887, "output_tokens": 2999380, "reasoning_tokens": 202288, "cache_read_tokens": 203585792, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 83.442, "latency": 656.357, "stderr": 0.447, "cost_per_test": 0.383981, "token_totals": { "input_tokens": 993049543, "output_tokens": 9388666, "reasoning_tokens": 2979345, "cache_read_tokens": 909992290, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 83.442, "latency": 1051.83, "stderr": 0.447, "cost_per_test": 0.303552, "token_totals": { "input_tokens": 1460813994, "output_tokens": 12095727, "reasoning_tokens": 6619071, "cache_read_tokens": 1422964599, "cache_write_tokens": 37829190 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 82.197, "latency": 3136.646, "stderr": 0.46, "cost_per_test": 0.570286, "token_totals": { "input_tokens": 3914223047, "output_tokens": 18369668, "reasoning_tokens": null, "cache_read_tokens": 3828746240, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 80.547, "latency": 1370.296, "stderr": 0.476, "cost_per_test": 10.374554, "token_totals": { "input_tokens": 1221154582, "output_tokens": 10565145, "reasoning_tokens": 2786144, "cache_read_tokens": 1150617952, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 80.504, "latency": 544.226, "stderr": 0.477, "cost_per_test": 0.107095, "token_totals": { "input_tokens": 224714337, "output_tokens": 3198839, "reasoning_tokens": 0, "cache_read_tokens": 203250560, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.5": { "accuracy": 79.983, "latency": 733.847, "stderr": 0.481, "cost_per_test": 4.597894, "token_totals": { "input_tokens": 372470134, "output_tokens": 4184434, "reasoning_tokens": 1267952, "cache_read_tokens": 341294080, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 79.751, "latency": 1024.886, "stderr": 0.483, "cost_per_test": 3.202399, "token_totals": { "input_tokens": 802970365, "output_tokens": 9301238, "reasoning_tokens": 2973597, "cache_read_tokens": 740704262, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 79.749, "latency": 888.433, "stderr": 0.486, "cost_per_test": 3.940956, "token_totals": { "input_tokens": 488567465, "output_tokens": 3914542, "reasoning_tokens": 154232, "cache_read_tokens": 0, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 79.317, "latency": 478.414, "stderr": 0.487, "cost_per_test": 2.116049, "token_totals": { "input_tokens": 606812300, "output_tokens": 8678382, "reasoning_tokens": 3806922, "cache_read_tokens": 543998500, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 78.188, "latency": 419.277, "stderr": 0.497, "cost_per_test": 0.036252, "token_totals": { "input_tokens": 252892290, "output_tokens": 3100583, "reasoning_tokens": 145588, "cache_read_tokens": 232673984, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 77.898, "latency": 358.218, "stderr": 0.499, "cost_per_test": 0.045991, "token_totals": { "input_tokens": 212025841, "output_tokens": 4296441, "reasoning_tokens": null, "cache_read_tokens": 166167424, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 75.77, "latency": 3175.128, "stderr": 0.539, "cost_per_test": 21.37949, "token_totals": { "input_tokens": 2462576678, "output_tokens": 38472452, "reasoning_tokens": 24925173, "cache_read_tokens": 2286257168, "cache_write_tokens": 176308820 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 74.497, "latency": 876.217, "stderr": 0.524, "cost_per_test": 3.598304, "token_totals": { "input_tokens": 665491305, "output_tokens": 5254641, "reasoning_tokens": 3014803, "cache_read_tokens": 623007616, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 74.063, "latency": 1405.253, "stderr": 0.527, "cost_per_test": 0.818741, "token_totals": { "input_tokens": 373247916, "output_tokens": 3422680, "reasoning_tokens": 0, "cache_read_tokens": 341806731, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 74.005, "latency": 1870.966, "stderr": 0.528, "cost_per_test": 0.174542, "token_totals": { "input_tokens": 201049757, "output_tokens": 5401207, "reasoning_tokens": null, "cache_read_tokens": 191047296, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 73.427, "latency": 2113.923, "stderr": 0.545, "cost_per_test": 3.394654, "token_totals": { "input_tokens": 4127341953, "output_tokens": 54990908, "reasoning_tokens": 37472036, "cache_read_tokens": 4002588737, "cache_write_tokens": 124738510 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 71.718, "latency": 498.074, "stderr": 0.542, "cost_per_test": 0.555636, "token_totals": { "input_tokens": 218148148, "output_tokens": 2889617, "reasoning_tokens": 207437, "cache_read_tokens": 200898240, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 71.689, "latency": 161.487, "stderr": 0.55, "cost_per_test": 0.046313, "token_totals": { "input_tokens": 146486877, "output_tokens": 4726655, "reasoning_tokens": null, "cache_read_tokens": 110128578, "cache_write_tokens": null }, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 69.952, "latency": 3464.426, "stderr": 0.552, "cost_per_test": 16.612583, "token_totals": { "input_tokens": 4750998856, "output_tokens": 53560846, "reasoning_tokens": 35373065, "cache_read_tokens": 4582495471, "cache_write_tokens": 168486127 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 69.532, "latency": 232.895, "stderr": 0.554, "cost_per_test": 0.331805, "token_totals": { "input_tokens": 234750879, "output_tokens": 3541635, "reasoning_tokens": 1058834, "cache_read_tokens": 146158528, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 69.168, "latency": 1508.275, "stderr": 0.575, "cost_per_test": 0.351359, "token_totals": { "input_tokens": 236174566, "output_tokens": 2527211, "reasoning_tokens": 124700, "cache_read_tokens": 219868250, "cache_write_tokens": null }, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 65.674, "latency": 1103.692, "stderr": 0.574, "cost_per_test": 1.252023, "token_totals": { "input_tokens": 575866616, "output_tokens": 7829496, "reasoning_tokens": null, "cache_read_tokens": 552512512, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 65.631, "latency": 2048.539, "stderr": 0.586, "cost_per_test": null, "token_totals": { "input_tokens": 305706389, "output_tokens": 3758581, "reasoning_tokens": null, "cache_read_tokens": 48583680, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "grok/grok-4.3": { "accuracy": 64.843, "latency": 168.721, "stderr": 0.574, "cost_per_test": 0.389144, "token_totals": { "input_tokens": 88472856, "output_tokens": 3683568, "reasoning_tokens": 464382, "cache_read_tokens": 71653723, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 62.694, "latency": 529.178, "stderr": 0.587, "cost_per_test": 0.234226, "token_totals": { "input_tokens": 214653131, "output_tokens": 1748791, "reasoning_tokens": 535291, "cache_read_tokens": 170857216, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 62.315, "latency": 3694.945, "stderr": 0.595, "cost_per_test": 6.074265, "token_totals": { "input_tokens": 439770514, "output_tokens": 5184631, "reasoning_tokens": null, "cache_read_tokens": null, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 59.922, "latency": 373.931, "stderr": 0.59, "cost_per_test": 0.489594, "token_totals": { "input_tokens": 194662499, "output_tokens": 2889064, "reasoning_tokens": 177130, "cache_read_tokens": 11540069, "cache_write_tokens": 18939 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 58.579, "latency": 681.27, "stderr": 0.668, "cost_per_test": 1.001493, "token_totals": { "input_tokens": 440584578, "output_tokens": 6595483, "reasoning_tokens": 5276888, "cache_read_tokens": 389590016, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 56.086, "latency": 436.69, "stderr": 0.597, "cost_per_test": 1.149757, "token_totals": { "input_tokens": 230732445, "output_tokens": 1350000, "reasoning_tokens": 503310, "cache_read_tokens": 204622317, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 54.851, "latency": 465.122, "stderr": 0.612, "cost_per_test": 0.527685, "token_totals": { "input_tokens": 547314889, "output_tokens": 1917329, "reasoning_tokens": 177714, "cache_read_tokens": 495461278, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 52.23, "latency": 363.008, "stderr": 0.636, "cost_per_test": 0.184287, "token_totals": { "input_tokens": 315810901, "output_tokens": 2855013, "reasoning_tokens": 1338430, "cache_read_tokens": 261179392, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 48.27, "latency": 361.116, "stderr": 0.601, "cost_per_test": 0.416609, "token_totals": { "input_tokens": 129616489, "output_tokens": 3945656, "reasoning_tokens": 242225, "cache_read_tokens": 114566187, "cache_write_tokens": 15031667 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 45.318, "latency": 202.513, "stderr": 0.599, "cost_per_test": 0.264375, "token_totals": { "input_tokens": 225950385, "output_tokens": 1424392, "reasoning_tokens": 580723, "cache_read_tokens": 190051973, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 44.088, "latency": 502.903, "stderr": 0.6, "cost_per_test": 0.986814, "token_totals": { "input_tokens": 248182901, "output_tokens": 1198831, "reasoning_tokens": 239427, "cache_read_tokens": 216554703, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 41.1, "latency": 1194.199, "stderr": 0.712, "cost_per_test": 0.284127, "token_totals": { "input_tokens": 322083675, "output_tokens": 1329814, "reasoning_tokens": 131024, "cache_read_tokens": 189201152, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "inception/mercury-2.5": { "accuracy": 40.903, "latency": 131.54, "stderr": 0.592, "cost_per_test": 0.192614, "token_totals": { "input_tokens": 159575776, "output_tokens": 1621389, "reasoning_tokens": 1524161, "cache_read_tokens": 55653173, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 33.391, "latency": 1736.203, "stderr": 0.567, "cost_per_test": 0.028287, "token_totals": { "input_tokens": 216862233, "output_tokens": 2963970, "reasoning_tokens": null, "cache_read_tokens": 201035952, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 29.614, "latency": 60.554, "stderr": 0.549, "cost_per_test": 0.084048, "token_totals": { "input_tokens": 160066368, "output_tokens": 769494, "reasoning_tokens": 360545, "cache_read_tokens": 138155856, "cache_write_tokens": null }, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "antitrust_competition": { "meta/muse_spark_1_1": { "accuracy": 40.0, "latency": 708.48, "stderr": 24.495, "cost_per_test": 0.782306, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 40.0, "latency": 880.526, "stderr": 24.495, "cost_per_test": 4.295226, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 40.0, "latency": 1432.75, "stderr": 24.495, "cost_per_test": 1.84335, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 20.0, "latency": 545.892, "stderr": 20.0, "cost_per_test": 2.679324, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 20.0, "latency": 857.572, "stderr": 20.0, "cost_per_test": 2.562941, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 20.0, "latency": 923.89, "stderr": 20.0, "cost_per_test": 2.724467, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 20.0, "latency": 1808.052, "stderr": 20.0, "cost_per_test": 3.522356, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.7": { "accuracy": 20.0, "latency": 2336.496, "stderr": 20.0, "cost_per_test": 10.647354, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 20.0, "latency": 2680.208, "stderr": 20.0, "cost_per_test": 1.955704, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-4-argon": { "accuracy": 20.0, "latency": 3686.716, "stderr": 20.0, "cost_per_test": 15.535189, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 62.2, "stderr": 0.0, "cost_per_test": 0.065573, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 116.346, "stderr": 0.0, "cost_per_test": 0.128388, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 143.374, "stderr": 0.0, "cost_per_test": 0.348081, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 154.07, "stderr": 0.0, "cost_per_test": 0.045437, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 186.102, "stderr": 0.0, "cost_per_test": 0.456896, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 236.172, "stderr": 0.0, "cost_per_test": 0.244675, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 283.84, "stderr": 0.0, "cost_per_test": 0.174565, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 324.616, "stderr": 0.0, "cost_per_test": 0.444883, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 394.614, "stderr": 0.0, "cost_per_test": 0.058407, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 402.678, "stderr": 0.0, "cost_per_test": 0.039112, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 409.394, "stderr": 0.0, "cost_per_test": 0.8503, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 412.08, "stderr": 0.0, "cost_per_test": 0.42338, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 444.684, "stderr": 0.0, "cost_per_test": 2.159387, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 466.394, "stderr": 0.0, "cost_per_test": 0.546152, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 515.756, "stderr": 0.0, "cost_per_test": 1.014086, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 517.982, "stderr": 0.0, "cost_per_test": 2.535802, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 526.666, "stderr": 0.0, "cost_per_test": 0.679439, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 550.106, "stderr": 0.0, "cost_per_test": 0.959043, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 573.104, "stderr": 0.0, "cost_per_test": 0.12027, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 651.426, "stderr": 0.0, "cost_per_test": 2.727866, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 652.364, "stderr": 0.0, "cost_per_test": 0.44471, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 669.612, "stderr": 0.0, "cost_per_test": 3.748755, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 723.782, "stderr": 0.0, "cost_per_test": 0.052986, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 724.042, "stderr": 0.0, "cost_per_test": 2.232444, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 746.252, "stderr": 0.0, "cost_per_test": 0.170347, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 753.94, "stderr": 0.0, "cost_per_test": 3.79389, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 754.36, "stderr": 0.0, "cost_per_test": 1.066461, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 809.932, "stderr": 0.0, "cost_per_test": 2.734005, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 933.252, "stderr": 0.0, "cost_per_test": 0.077418, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 977.044, "stderr": 0.0, "cost_per_test": 0.192211, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 994.292, "stderr": 0.0, "cost_per_test": 2.673062, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 1000.294, "stderr": 0.0, "cost_per_test": 0.706227, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1008.986, "stderr": 0.0, "cost_per_test": 1.357335, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 1009.726, "stderr": 0.0, "cost_per_test": 3.466604, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 1057.978, "stderr": 0.0, "cost_per_test": 1.668923, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1062.228, "stderr": 0.0, "cost_per_test": 0.153299, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1101.62, "stderr": 0.0, "cost_per_test": 2.027261, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1172.084, "stderr": 0.0, "cost_per_test": 8.940461, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1296.742, "stderr": 0.0, "cost_per_test": 0.809466, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1301.104, "stderr": 0.0, "cost_per_test": 0.768634, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1311.106, "stderr": 0.0, "cost_per_test": 9.124408, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1337.142, "stderr": 0.0, "cost_per_test": 9.296892, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 1341.73, "stderr": 0.0, "cost_per_test": 0.320155, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1344.04, "stderr": 0.0, "cost_per_test": 1.499283, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1474.888, "stderr": 0.0, "cost_per_test": 24.979276, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1501.212, "stderr": 0.0, "cost_per_test": 16.832828, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1519.004, "stderr": 0.0, "cost_per_test": 0.789632, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 1839.778, "stderr": 0.0, "cost_per_test": 3.871498, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1972.514, "stderr": 0.0, "cost_per_test": 0.192669, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 2025.322, "stderr": 0.0, "cost_per_test": 6.844893, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 2051.11, "stderr": 0.0, "cost_per_test": 0.415698, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 2090.638, "stderr": 0.0, "cost_per_test": 3.909019, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2112.382, "stderr": 0.0, "cost_per_test": 3.206948, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 2246.048, "stderr": 0.0, "cost_per_test": 2.392755, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2421.988, "stderr": 0.0, "cost_per_test": 3.497776, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 2430.912, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 2490.34, "stderr": 0.0, "cost_per_test": 0.17757, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 2522.454, "stderr": 0.0, "cost_per_test": 3.772087, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 2610.216, "stderr": 0.0, "cost_per_test": 2.722361, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 2632.506, "stderr": 0.0, "cost_per_test": 19.151916, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 2674.686, "stderr": 0.0, "cost_per_test": 0.45672, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3271.126, "stderr": 0.0, "cost_per_test": 23.046646, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 3340.984, "stderr": 0.0, "cost_per_test": 0.022637, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3503.03, "stderr": 0.0, "cost_per_test": 17.847147, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 4346.874, "stderr": 0.0, "cost_per_test": 6.441935, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 6352.144, "stderr": 0.0, "cost_per_test": 39.631516, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "arbitration_international_dispute_resolution": { "meta/muse_spark_1_3_max": { "accuracy": 20.0, "latency": 931.43, "stderr": 20.0, "cost_per_test": 3.426857, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 20.0, "latency": 961.242, "stderr": 20.0, "cost_per_test": 0.098258, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 20.0, "latency": 1617.612, "stderr": 20.0, "cost_per_test": 2.621978, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": 2733.414, "stderr": 20.0, "cost_per_test": 6.270327, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 85.232, "stderr": 0.0, "cost_per_test": 0.079025, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 151.01, "stderr": 0.0, "cost_per_test": 0.294423, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 164.272, "stderr": 0.0, "cost_per_test": 0.052176, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 186.976, "stderr": 0.0, "cost_per_test": 0.346871, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 199.3, "stderr": 0.0, "cost_per_test": 0.354648, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 226.358, "stderr": 0.0, "cost_per_test": 0.288514, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 240.988, "stderr": 0.0, "cost_per_test": 0.255694, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 314.632, "stderr": 0.0, "cost_per_test": 2.222368, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 323.41, "stderr": 0.0, "cost_per_test": 0.225776, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 335.17, "stderr": 0.0, "cost_per_test": 0.370039, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 386.628, "stderr": 0.0, "cost_per_test": 0.042567, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 390.77, "stderr": 0.0, "cost_per_test": 0.048701, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 395.842, "stderr": 0.0, "cost_per_test": 0.887292, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 412.718, "stderr": 0.0, "cost_per_test": 0.513055, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 431.252, "stderr": 0.0, "cost_per_test": 0.563167, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 499.18, "stderr": 0.0, "cost_per_test": 0.123822, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 502.534, "stderr": 0.0, "cost_per_test": 1.759751, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 553.876, "stderr": 0.0, "cost_per_test": 0.210863, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 573.844, "stderr": 0.0, "cost_per_test": 2.445749, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 589.422, "stderr": 0.0, "cost_per_test": 0.801309, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 643.878, "stderr": 0.0, "cost_per_test": 4.788394, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 644.176, "stderr": 0.0, "cost_per_test": 0.879378, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 664.974, "stderr": 0.0, "cost_per_test": 0.232631, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 676.282, "stderr": 0.0, "cost_per_test": 0.513603, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 679.778, "stderr": 0.0, "cost_per_test": 7.105465, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 704.482, "stderr": 0.0, "cost_per_test": 0.990325, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 0.0, "latency": 774.504, "stderr": 0.0, "cost_per_test": 0.973952, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 799.21, "stderr": 0.0, "cost_per_test": 1.403604, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 828.34, "stderr": 0.0, "cost_per_test": 6.264231, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 854.154, "stderr": 0.0, "cost_per_test": 1.667616, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 926.49, "stderr": 0.0, "cost_per_test": 0.076998, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 940.9, "stderr": 0.0, "cost_per_test": 4.937446, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": 1037.384, "stderr": 0.0, "cost_per_test": 4.168125, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1065.36, "stderr": 0.0, "cost_per_test": 0.22831, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1067.75, "stderr": 0.0, "cost_per_test": 3.767811, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1113.428, "stderr": 0.0, "cost_per_test": 0.504216, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 1162.808, "stderr": 0.0, "cost_per_test": 3.307162, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1188.18, "stderr": 0.0, "cost_per_test": 0.91661, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 1231.44, "stderr": 0.0, "cost_per_test": 5.708221, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1346.446, "stderr": 0.0, "cost_per_test": 2.719674, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 1350.512, "stderr": 0.0, "cost_per_test": 37.520705, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 1361.618, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1371.892, "stderr": 0.0, "cost_per_test": 17.046465, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1441.47, "stderr": 0.0, "cost_per_test": 39.13627, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1522.162, "stderr": 0.0, "cost_per_test": 0.220117, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1580.326, "stderr": 0.0, "cost_per_test": 12.077574, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1598.092, "stderr": 0.0, "cost_per_test": 13.097728, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1678.318, "stderr": 0.0, "cost_per_test": 1.100251, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1694.128, "stderr": 0.0, "cost_per_test": 0.429547, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1715.52, "stderr": 0.0, "cost_per_test": 22.492624, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1787.716, "stderr": 0.0, "cost_per_test": 1.332675, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 1798.27, "stderr": 0.0, "cost_per_test": 1.869712, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 1801.3, "stderr": 0.0, "cost_per_test": 0.172169, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1866.306, "stderr": 0.0, "cost_per_test": 2.296246, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 1912.688, "stderr": 0.0, "cost_per_test": 3.164122, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 1993.245, "stderr": 0.0, "cost_per_test": 0.385748, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 1998.744, "stderr": 0.0, "cost_per_test": 11.650358, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 0.0, "latency": 2088.364, "stderr": 0.0, "cost_per_test": 4.442233, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 2172.626, "stderr": 0.0, "cost_per_test": 4.415408, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2297.508, "stderr": 0.0, "cost_per_test": 14.639679, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2429.578, "stderr": 0.0, "cost_per_test": 5.683671, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 2578.722, "stderr": 0.0, "cost_per_test": 4.266851, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 2582.258, "stderr": 0.0, "cost_per_test": 12.524633, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 2729.106, "stderr": 0.0, "cost_per_test": 21.027612, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 2738.558, "stderr": 0.0, "cost_per_test": 5.272515, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 2837.648, "stderr": 0.0, "cost_per_test": 0.057252, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3259.595, "stderr": 0.0, "cost_per_test": 22.280508, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3597.47, "stderr": 0.0, "cost_per_test": 16.01159, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 4238.472, "stderr": 0.0, "cost_per_test": 0.842934, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 4724.642, "stderr": 0.0, "cost_per_test": 3.530728, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 5956.935, "stderr": 0.0, "cost_per_test": 8.978748, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 14982.956, "stderr": 0.0, "cost_per_test": 93.432713, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "banking_finance": { "google/gemini-3.7-flash": { "accuracy": 20.0, "latency": 435.288, "stderr": 20.0, "cost_per_test": 2.07755, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 20.0, "latency": 666.63, "stderr": 20.0, "cost_per_test": 2.360297, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 20.0, "latency": 694.004, "stderr": 20.0, "cost_per_test": 2.511915, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 20.0, "latency": 1239.682, "stderr": 20.0, "cost_per_test": 1.815007, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 20.0, "latency": 2610.074, "stderr": 20.0, "cost_per_test": 3.236722, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 49.89, "stderr": 0.0, "cost_per_test": 0.057659, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 110.462, "stderr": 0.0, "cost_per_test": 0.223979, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 124.376, "stderr": 0.0, "cost_per_test": 0.220183, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 132.518, "stderr": 0.0, "cost_per_test": 0.047389, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 188.502, "stderr": 0.0, "cost_per_test": 0.160961, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 189.08, "stderr": 0.0, "cost_per_test": 0.12378, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 206.032, "stderr": 0.0, "cost_per_test": 0.030466, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 233.378, "stderr": 0.0, "cost_per_test": 0.356049, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 297.212, "stderr": 0.0, "cost_per_test": 0.49634, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 306.848, "stderr": 0.0, "cost_per_test": 0.674698, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 311.896, "stderr": 0.0, "cost_per_test": 0.609606, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 323.646, "stderr": 0.0, "cost_per_test": 0.132613, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 335.918, "stderr": 0.0, "cost_per_test": 0.353865, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 343.366, "stderr": 0.0, "cost_per_test": 0.033503, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 363.528, "stderr": 0.0, "cost_per_test": 0.694725, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 371.812, "stderr": 0.0, "cost_per_test": 0.486311, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 413.544, "stderr": 0.0, "cost_per_test": 0.507355, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 440.778, "stderr": 0.0, "cost_per_test": 1.899233, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 461.856, "stderr": 0.0, "cost_per_test": 0.095159, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 462.162, "stderr": 0.0, "cost_per_test": 1.30437, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 493.128, "stderr": 0.0, "cost_per_test": 0.258578, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 0.0, "latency": 553.184, "stderr": 0.0, "cost_per_test": 0.664525, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 559.192, "stderr": 0.0, "cost_per_test": 0.939858, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 579.452, "stderr": 0.0, "cost_per_test": 3.830095, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 632.108, "stderr": 0.0, "cost_per_test": 3.453057, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 638.052, "stderr": 0.0, "cost_per_test": 0.053598, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": 698.168, "stderr": 0.0, "cost_per_test": 1.749007, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 708.482, "stderr": 0.0, "cost_per_test": 1.580358, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 715.268, "stderr": 0.0, "cost_per_test": 1.805379, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 737.218, "stderr": 0.0, "cost_per_test": 2.745671, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 766.152, "stderr": 0.0, "cost_per_test": 0.171268, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 781.298, "stderr": 0.0, "cost_per_test": 2.221578, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 784.634, "stderr": 0.0, "cost_per_test": 3.559689, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 830.14, "stderr": 0.0, "cost_per_test": 0.071494, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 848.694, "stderr": 0.0, "cost_per_test": 0.318436, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 866.12, "stderr": 0.0, "cost_per_test": 0.183967, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 866.176, "stderr": 0.0, "cost_per_test": 3.089661, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 982.432, "stderr": 0.0, "cost_per_test": 2.744751, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1096.08, "stderr": 0.0, "cost_per_test": 1.070964, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1130.346, "stderr": 0.0, "cost_per_test": 8.75203, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1180.702, "stderr": 0.0, "cost_per_test": 1.170507, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1245.96, "stderr": 0.0, "cost_per_test": 9.097239, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1258.57, "stderr": 0.0, "cost_per_test": 0.160708, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1264.306, "stderr": 0.0, "cost_per_test": 2.104294, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 1283.442, "stderr": 0.0, "cost_per_test": 0.0228, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1299.434, "stderr": 0.0, "cost_per_test": 20.419711, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1314.3, "stderr": 0.0, "cost_per_test": 0.794833, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1390.374, "stderr": 0.0, "cost_per_test": 15.803642, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 1398.295, "stderr": 0.0, "cost_per_test": 0.219791, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1512.716, "stderr": 0.0, "cost_per_test": 0.334669, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1562.144, "stderr": 0.0, "cost_per_test": 10.733228, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1611.048, "stderr": 0.0, "cost_per_test": 0.82827, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 1633.274, "stderr": 0.0, "cost_per_test": 1.681982, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 1658.056, "stderr": 0.0, "cost_per_test": 8.036238, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1663.1, "stderr": 0.0, "cost_per_test": 0.858064, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 1770.286, "stderr": 0.0, "cost_per_test": 1.981182, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 1895.866, "stderr": 0.0, "cost_per_test": 0.154828, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2008.814, "stderr": 0.0, "cost_per_test": 3.278736, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2139.946, "stderr": 0.0, "cost_per_test": 3.837556, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 2147.02, "stderr": 0.0, "cost_per_test": 8.507105, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 2244.842, "stderr": 0.0, "cost_per_test": 3.38767, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 2253.744, "stderr": 0.0, "cost_per_test": 3.51918, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 2267.098, "stderr": 0.0, "cost_per_test": 5.116919, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 2380.224, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2471.01, "stderr": 0.0, "cost_per_test": 8.387325, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 2922.154, "stderr": 0.0, "cost_per_test": 3.105232, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3006.784, "stderr": 0.0, "cost_per_test": 23.532694, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 3281.074, "stderr": 0.0, "cost_per_test": 0.617318, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3289.418, "stderr": 0.0, "cost_per_test": 16.890636, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 3388.15, "stderr": 0.0, "cost_per_test": 24.310288, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 3993.99, "stderr": 0.0, "cost_per_test": 35.04233, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "bankruptcy_restructuring": { "meta/muse_spark_1_3": { "accuracy": 40.0, "latency": 909.652, "stderr": 24.495, "cost_per_test": 2.123925, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 20.0, "latency": 817.014, "stderr": 20.0, "cost_per_test": 5.003405, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 20.0, "latency": 838.64, "stderr": 20.0, "cost_per_test": 0.878241, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 53.972, "stderr": 0.0, "cost_per_test": 0.111354, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 108.948, "stderr": 0.0, "cost_per_test": 0.341968, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 165.814, "stderr": 0.0, "cost_per_test": 0.462489, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 170.688, "stderr": 0.0, "cost_per_test": 0.234753, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 200.918, "stderr": 0.0, "cost_per_test": 0.239951, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 202.907, "stderr": 0.0, "cost_per_test": 0.272626, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 217.848, "stderr": 0.0, "cost_per_test": 0.833477, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 218.866, "stderr": 0.0, "cost_per_test": 0.062508, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 252.102, "stderr": 0.0, "cost_per_test": 0.604154, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 265.024, "stderr": 0.0, "cost_per_test": 0.35207, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 301.174, "stderr": 0.0, "cost_per_test": 0.043424, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 331.954, "stderr": 0.0, "cost_per_test": 0.037577, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 358.702, "stderr": 0.0, "cost_per_test": 3.528761, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 377.686, "stderr": 0.0, "cost_per_test": 0.705278, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 392.87, "stderr": 0.0, "cost_per_test": 0.794801, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 457.39, "stderr": 0.0, "cost_per_test": 0.62914, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 461.044, "stderr": 0.0, "cost_per_test": 0.269306, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 473.106, "stderr": 0.0, "cost_per_test": 0.106362, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 476.416, "stderr": 0.0, "cost_per_test": 0.041537, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 480.904, "stderr": 0.0, "cost_per_test": 0.181559, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 494.646, "stderr": 0.0, "cost_per_test": 2.21075, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 570.796, "stderr": 0.0, "cost_per_test": 2.356863, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 624.686, "stderr": 0.0, "cost_per_test": 0.211492, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 663.352, "stderr": 0.0, "cost_per_test": 0.549745, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 738.64, "stderr": 0.0, "cost_per_test": 4.430016, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 805.67, "stderr": 0.0, "cost_per_test": 3.843686, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 811.427, "stderr": 0.0, "cost_per_test": 1.062026, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 881.327, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 897.978, "stderr": 0.0, "cost_per_test": 2.382899, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 913.392, "stderr": 0.0, "cost_per_test": 0.351571, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 940.394, "stderr": 0.0, "cost_per_test": 4.66934, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 959.834, "stderr": 0.0, "cost_per_test": 0.310916, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 991.99, "stderr": 0.0, "cost_per_test": 3.06916, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 1015.696, "stderr": 0.0, "cost_per_test": 5.355597, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1073.03, "stderr": 0.0, "cost_per_test": 0.35608, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1132.804, "stderr": 0.0, "cost_per_test": 3.545522, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 0.0, "latency": 1146.54, "stderr": 0.0, "cost_per_test": 1.980681, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 1149.696, "stderr": 0.0, "cost_per_test": 2.792828, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1250.194, "stderr": 0.0, "cost_per_test": 0.309228, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 0.0, "latency": 1271.85, "stderr": 0.0, "cost_per_test": 2.982148, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 1299.734, "stderr": 0.0, "cost_per_test": 0.116291, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1343.498, "stderr": 0.0, "cost_per_test": 1.508844, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1356.36, "stderr": 0.0, "cost_per_test": 1.383205, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1383.638, "stderr": 0.0, "cost_per_test": 0.186039, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1398.952, "stderr": 0.0, "cost_per_test": 11.784326, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1470.566, "stderr": 0.0, "cost_per_test": 1.001911, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1473.246, "stderr": 0.0, "cost_per_test": 13.525029, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1531.122, "stderr": 0.0, "cost_per_test": 2.87069, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1540.87, "stderr": 0.0, "cost_per_test": 12.196515, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1593.86, "stderr": 0.0, "cost_per_test": 37.668899, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1683.26, "stderr": 0.0, "cost_per_test": 23.012162, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 1767.562, "stderr": 0.0, "cost_per_test": 0.191614, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1782.292, "stderr": 0.0, "cost_per_test": 1.021346, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1831.932, "stderr": 0.0, "cost_per_test": 1.145459, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 1962.126, "stderr": 0.0, "cost_per_test": 4.332858, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 2030.7, "stderr": 0.0, "cost_per_test": 10.336011, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2176.036, "stderr": 0.0, "cost_per_test": 3.717602, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 2251.578, "stderr": 0.0, "cost_per_test": 0.069982, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 2252.374, "stderr": 0.0, "cost_per_test": 2.436706, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 0.0, "latency": 2433.718, "stderr": 0.0, "cost_per_test": 5.299975, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2529.162, "stderr": 0.0, "cost_per_test": 13.506458, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 2535.254, "stderr": 0.0, "cost_per_test": 4.663981, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 2552.876, "stderr": 0.0, "cost_per_test": 5.241005, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 2664.496, "stderr": 0.0, "cost_per_test": 4.724615, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 2768.728, "stderr": 0.0, "cost_per_test": 22.051917, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 2967.024, "stderr": 0.0, "cost_per_test": 15.129321, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 2999.574, "stderr": 0.0, "cost_per_test": 25.189219, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 3302.308, "stderr": 0.0, "cost_per_test": 11.627608, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 3939.598, "stderr": 0.0, "cost_per_test": 5.799978, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 4087.77, "stderr": 0.0, "cost_per_test": 6.094256, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 5011.65, "stderr": 0.0, "cost_per_test": 0.870454, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 8020.762, "stderr": 0.0, "cost_per_test": 3.577614, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 10274.438, "stderr": 0.0, "cost_per_test": 87.544464, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "capital_markets": { "meta/muse_spark_1_2": { "accuracy": 60.0, "latency": 960.29, "stderr": 24.495, "cost_per_test": 1.766404, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.5": { "accuracy": 40.0, "latency": 544.732, "stderr": 24.495, "cost_per_test": 1.23096, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 40.0, "latency": 707.676, "stderr": 24.495, "cost_per_test": 1.784133, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 40.0, "latency": 1082.806, "stderr": 24.495, "cost_per_test": 2.051823, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 40.0, "latency": 2118.234, "stderr": 24.495, "cost_per_test": 9.03284, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 40.0, "latency": 2502.272, "stderr": 24.495, "cost_per_test": 9.722399, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 20.0, "latency": 322.07, "stderr": 20.0, "cost_per_test": 2.854243, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 20.0, "latency": 331.896, "stderr": 20.0, "cost_per_test": 0.366377, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 20.0, "latency": 350.396, "stderr": 20.0, "cost_per_test": 0.764042, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 20.0, "latency": 499.572, "stderr": 20.0, "cost_per_test": 3.218847, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 20.0, "latency": 557.858, "stderr": 20.0, "cost_per_test": 0.645246, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 20.0, "latency": 586.44, "stderr": 20.0, "cost_per_test": 2.819389, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 20.0, "latency": 634.18, "stderr": 20.0, "cost_per_test": 2.399124, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 20.0, "latency": 688.528, "stderr": 20.0, "cost_per_test": 1.181787, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 20.0, "latency": 923.596, "stderr": 20.0, "cost_per_test": 2.396947, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 20.0, "latency": 931.196, "stderr": 20.0, "cost_per_test": 0.941373, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 20.0, "latency": 963.78, "stderr": 20.0, "cost_per_test": 1.900859, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 20.0, "latency": 1228.556, "stderr": 20.0, "cost_per_test": 10.574556, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 20.0, "latency": 1231.11, "stderr": 20.0, "cost_per_test": 9.299876, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 20.0, "latency": 1453.458, "stderr": 20.0, "cost_per_test": 0.150847, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "stepfun/step-5-preview": { "accuracy": 20.0, "latency": 1604.092, "stderr": 20.0, "cost_per_test": 0.917971, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 35.432, "stderr": 0.0, "cost_per_test": 0.043334, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 64.014, "stderr": 0.0, "cost_per_test": 0.121463, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 97.31, "stderr": 0.0, "cost_per_test": 0.251837, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 122.554, "stderr": 0.0, "cost_per_test": 0.036275, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 125.772, "stderr": 0.0, "cost_per_test": 0.232911, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 188.528, "stderr": 0.0, "cost_per_test": 0.261643, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 250.502, "stderr": 0.0, "cost_per_test": 0.031194, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 255.418, "stderr": 0.0, "cost_per_test": 0.119538, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 257.284, "stderr": 0.0, "cost_per_test": 0.735522, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 263.832, "stderr": 0.0, "cost_per_test": 0.291257, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 327.828, "stderr": 0.0, "cost_per_test": 0.415094, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 347.665, "stderr": 0.0, "cost_per_test": 0.13166, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 374.008, "stderr": 0.0, "cost_per_test": 0.031216, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 393.834, "stderr": 0.0, "cost_per_test": 0.443784, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 397.338, "stderr": 0.0, "cost_per_test": 0.490152, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 413.782, "stderr": 0.0, "cost_per_test": 1.728373, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 443.016, "stderr": 0.0, "cost_per_test": 0.110985, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 456.474, "stderr": 0.0, "cost_per_test": 0.16244, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 507.738, "stderr": 0.0, "cost_per_test": 0.283023, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 578.198, "stderr": 0.0, "cost_per_test": 3.800605, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 626.825, "stderr": 0.0, "cost_per_test": 0.755231, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 678.396, "stderr": 0.0, "cost_per_test": 3.277627, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 692.736, "stderr": 0.0, "cost_per_test": 1.167551, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 721.686, "stderr": 0.0, "cost_per_test": 2.678383, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 740.832, "stderr": 0.0, "cost_per_test": 2.617447, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 766.514, "stderr": 0.0, "cost_per_test": 0.073201, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 788.504, "stderr": 0.0, "cost_per_test": 0.197475, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 950.504, "stderr": 0.0, "cost_per_test": 0.018306, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 962.008, "stderr": 0.0, "cost_per_test": 0.329241, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 975.162, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1055.51, "stderr": 0.0, "cost_per_test": 1.763195, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1084.768, "stderr": 0.0, "cost_per_test": 17.303719, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1088.286, "stderr": 0.0, "cost_per_test": 0.136395, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1100.542, "stderr": 0.0, "cost_per_test": 0.64382, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 1121.472, "stderr": 0.0, "cost_per_test": 0.048956, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1236.618, "stderr": 0.0, "cost_per_test": 0.197788, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1245.51, "stderr": 0.0, "cost_per_test": 5.036098, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 1292.068, "stderr": 0.0, "cost_per_test": 5.128468, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 1347.776, "stderr": 0.0, "cost_per_test": 2.151201, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1359.526, "stderr": 0.0, "cost_per_test": 17.509413, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1403.207, "stderr": 0.0, "cost_per_test": 0.284413, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1460.304, "stderr": 0.0, "cost_per_test": 0.791921, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 1598.592, "stderr": 0.0, "cost_per_test": 1.415374, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 1692.948, "stderr": 0.0, "cost_per_test": 3.089995, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 1787.54, "stderr": 0.0, "cost_per_test": 3.174123, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 1907.572, "stderr": 0.0, "cost_per_test": 2.375674, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 2210.02, "stderr": 0.0, "cost_per_test": 0.433283, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.6": { "accuracy": 0.0, "latency": 2226.112, "stderr": 0.0, "cost_per_test": 3.540513, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2231.008, "stderr": 0.0, "cost_per_test": 3.463263, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 2330.642, "stderr": 0.0, "cost_per_test": 18.356578, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 2332.704, "stderr": 0.0, "cost_per_test": 1.737948, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 2665.926, "stderr": 0.0, "cost_per_test": 32.845013, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 2856.812, "stderr": 0.0, "cost_per_test": 14.684627, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 2863.46, "stderr": 0.0, "cost_per_test": 20.393428, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 5620.726, "stderr": 0.0, "cost_per_test": 4.981585, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null } }, "corporate_governance": { "meta/muse_spark_1_3_max": { "accuracy": 60.0, "latency": 1143.856, "stderr": 24.495, "cost_per_test": 2.672757, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 40.0, "latency": 874.242, "stderr": 24.495, "cost_per_test": 2.728836, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 20.0, "latency": 329.304, "stderr": 20.0, "cost_per_test": 3.011145, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 20.0, "latency": 736.924, "stderr": 20.0, "cost_per_test": 0.07785, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 20.0, "latency": 1321.064, "stderr": 20.0, "cost_per_test": 1.030941, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 20.0, "latency": 1426.99, "stderr": 20.0, "cost_per_test": 2.15945, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 20.0, "latency": 1644.848, "stderr": 20.0, "cost_per_test": 18.661957, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 20.0, "latency": 2166.744, "stderr": 20.0, "cost_per_test": 5.377027, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 20.0, "latency": 2442.838, "stderr": 20.0, "cost_per_test": 7.288851, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 20.0, "latency": 3315.052, "stderr": 20.0, "cost_per_test": 4.21486, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 45.132, "stderr": 0.0, "cost_per_test": 0.053496, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 105.77, "stderr": 0.0, "cost_per_test": 0.271182, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 139.304, "stderr": 0.0, "cost_per_test": 0.392709, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 161.014, "stderr": 0.0, "cost_per_test": 0.042448, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 166.626, "stderr": 0.0, "cost_per_test": 0.171496, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 174.342, "stderr": 0.0, "cost_per_test": 0.394478, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 298.072, "stderr": 0.0, "cost_per_test": 1.357341, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 355.344, "stderr": 0.0, "cost_per_test": 0.585215, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 364.926, "stderr": 0.0, "cost_per_test": 0.133065, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 372.972, "stderr": 0.0, "cost_per_test": 0.04276, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 386.21, "stderr": 0.0, "cost_per_test": 0.034168, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 422.362, "stderr": 0.0, "cost_per_test": 0.495369, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 432.855, "stderr": 0.0, "cost_per_test": 0.156264, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 437.184, "stderr": 0.0, "cost_per_test": 0.458097, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 445.236, "stderr": 0.0, "cost_per_test": 0.63414, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 464.67, "stderr": 0.0, "cost_per_test": 0.55015, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 465.482, "stderr": 0.0, "cost_per_test": 0.593195, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 506.593, "stderr": 0.0, "cost_per_test": 0.786027, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 518.074, "stderr": 0.0, "cost_per_test": 2.188345, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 543.246, "stderr": 0.0, "cost_per_test": 3.202037, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 617.67, "stderr": 0.0, "cost_per_test": 0.095359, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 669.87, "stderr": 0.0, "cost_per_test": 1.86509, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 676.244, "stderr": 0.0, "cost_per_test": 0.182687, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 693.504, "stderr": 0.0, "cost_per_test": 0.327343, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 754.62, "stderr": 0.0, "cost_per_test": 4.210018, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 757.952, "stderr": 0.0, "cost_per_test": 6.425704, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 807.578, "stderr": 0.0, "cost_per_test": 3.489705, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 812.582, "stderr": 0.0, "cost_per_test": 1.792585, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 854.396, "stderr": 0.0, "cost_per_test": 0.246185, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 895.086, "stderr": 0.0, "cost_per_test": 3.705633, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 935.4, "stderr": 0.0, "cost_per_test": 0.202579, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 937.754, "stderr": 0.0, "cost_per_test": 1.394328, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 960.338, "stderr": 0.0, "cost_per_test": 0.033144, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1194.222, "stderr": 0.0, "cost_per_test": 0.37409, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1311.706, "stderr": 0.0, "cost_per_test": 0.826881, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1323.154, "stderr": 0.0, "cost_per_test": 3.276748, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1349.544, "stderr": 0.0, "cost_per_test": 10.906051, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1380.374, "stderr": 0.0, "cost_per_test": 2.394515, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1402.604, "stderr": 0.0, "cost_per_test": 1.37422, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1409.04, "stderr": 0.0, "cost_per_test": 0.174308, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1412.708, "stderr": 0.0, "cost_per_test": 11.140007, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1546.64, "stderr": 0.0, "cost_per_test": 0.874361, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 1552.306, "stderr": 0.0, "cost_per_test": 4.414299, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 1552.808, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1554.822, "stderr": 0.0, "cost_per_test": 9.763798, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1583.514, "stderr": 0.0, "cost_per_test": 33.095665, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1606.784, "stderr": 0.0, "cost_per_test": 1.493244, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 1688.058, "stderr": 0.0, "cost_per_test": 0.159111, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1740.68, "stderr": 0.0, "cost_per_test": 1.030237, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 1879.302, "stderr": 0.0, "cost_per_test": 0.069229, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 1958.866, "stderr": 0.0, "cost_per_test": 10.888301, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 1980.132, "stderr": 0.0, "cost_per_test": 1.221399, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2145.662, "stderr": 0.0, "cost_per_test": 3.673218, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 2324.97, "stderr": 0.0, "cost_per_test": 0.445506, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 2334.13, "stderr": 0.0, "cost_per_test": 5.247151, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 2376.882, "stderr": 0.0, "cost_per_test": 3.186137, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2434.85, "stderr": 0.0, "cost_per_test": 3.975993, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2513.988, "stderr": 0.0, "cost_per_test": 10.269134, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 2832.07, "stderr": 0.0, "cost_per_test": 4.666701, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3385.064, "stderr": 0.0, "cost_per_test": 24.770063, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3464.234, "stderr": 0.0, "cost_per_test": 16.225332, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 3550.886, "stderr": 0.0, "cost_per_test": 23.927034, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 3929.774, "stderr": 0.0, "cost_per_test": 2.441136, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 4249.13, "stderr": 0.0, "cost_per_test": 0.906277, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 4642.184, "stderr": 0.0, "cost_per_test": 6.850674, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 10823.312, "stderr": 0.0, "cost_per_test": 61.982943, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "corporate_ma": { "deepseek/deepseek-v4.1-flash": { "accuracy": 20.0, "latency": 444.366, "stderr": 20.0, "cost_per_test": 0.166084, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 20.0, "latency": 451.45, "stderr": 20.0, "cost_per_test": 0.855036, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 20.0, "latency": 597.146, "stderr": 20.0, "cost_per_test": 0.050437, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 20.0, "latency": 778.878, "stderr": 20.0, "cost_per_test": 0.847903, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 20.0, "latency": 782.066, "stderr": 20.0, "cost_per_test": 1.932419, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 20.0, "latency": 949.974, "stderr": 20.0, "cost_per_test": 2.35844, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.2": { "accuracy": 20.0, "latency": 1002.308, "stderr": 20.0, "cost_per_test": 1.650999, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 20.0, "latency": 1088.096, "stderr": 20.0, "cost_per_test": 0.144414, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 20.0, "latency": 1091.8, "stderr": 20.0, "cost_per_test": 0.332689, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 20.0, "latency": 1150.196, "stderr": 20.0, "cost_per_test": 0.07413, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 20.0, "latency": 1265.256, "stderr": 20.0, "cost_per_test": 1.735278, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 20.0, "latency": 1344.032, "stderr": 20.0, "cost_per_test": 10.138066, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 20.0, "latency": 1454.082, "stderr": 20.0, "cost_per_test": 0.778644, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 20.0, "latency": 1471.018, "stderr": 20.0, "cost_per_test": 18.410087, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 20.0, "latency": 1632.694, "stderr": 20.0, "cost_per_test": 0.94797, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-astra": { "accuracy": 20.0, "latency": 1816.776, "stderr": 20.0, "cost_per_test": 28.643676, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 20.0, "latency": 2112.8, "stderr": 20.0, "cost_per_test": 2.209117, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-4-argon": { "accuracy": 20.0, "latency": 2234.264, "stderr": 20.0, "cost_per_test": 8.132612, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 20.0, "latency": 2302.844, "stderr": 20.0, "cost_per_test": 3.938856, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 2461.11, "stderr": 20.0, "cost_per_test": 0.176755, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 20.0, "latency": 2657.59, "stderr": 20.0, "cost_per_test": 13.092829, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 20.0, "latency": 2906.61, "stderr": 20.0, "cost_per_test": 0.512019, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 20.0, "latency": 3062.552, "stderr": 20.0, "cost_per_test": 4.419748, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 20.0, "latency": 3381.568, "stderr": 20.0, "cost_per_test": 4.118716, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 20.0, "latency": 3391.348, "stderr": 20.0, "cost_per_test": 25.998286, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 20.0, "latency": 4606.318, "stderr": 20.0, "cost_per_test": 3.767001, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 68.164, "stderr": 0.0, "cost_per_test": 0.091726, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 97.04, "stderr": 0.0, "cost_per_test": 0.197043, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 130.574, "stderr": 0.0, "cost_per_test": 0.042211, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 134.364, "stderr": 0.0, "cost_per_test": 0.246824, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 189.98, "stderr": 0.0, "cost_per_test": 0.464081, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 198.466, "stderr": 0.0, "cost_per_test": 0.9394, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 239.62, "stderr": 0.0, "cost_per_test": 0.475678, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 248.88, "stderr": 0.0, "cost_per_test": 0.330445, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 305.354, "stderr": 0.0, "cost_per_test": 2.25119, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 316.128, "stderr": 0.0, "cost_per_test": 0.414307, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 360.64, "stderr": 0.0, "cost_per_test": 0.035857, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 375.644, "stderr": 0.0, "cost_per_test": 0.438626, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 412.996, "stderr": 0.0, "cost_per_test": 0.25487, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 421.036, "stderr": 0.0, "cost_per_test": 1.877373, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 457.828, "stderr": 0.0, "cost_per_test": 0.17506, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 458.588, "stderr": 0.0, "cost_per_test": 0.05375, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 499.322, "stderr": 0.0, "cost_per_test": 0.719898, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 499.4, "stderr": 0.0, "cost_per_test": 0.580534, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 505.846, "stderr": 0.0, "cost_per_test": 0.104926, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 531.144, "stderr": 0.0, "cost_per_test": 3.283063, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 685.316, "stderr": 0.0, "cost_per_test": 0.396871, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 748.426, "stderr": 0.0, "cost_per_test": 1.599282, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 767.346, "stderr": 0.0, "cost_per_test": 2.416866, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 799.498, "stderr": 0.0, "cost_per_test": 3.550379, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 803.612, "stderr": 0.0, "cost_per_test": 3.395472, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 804.433, "stderr": 0.0, "cost_per_test": 1.456426, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 832.006, "stderr": 0.0, "cost_per_test": 2.104291, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 871.134, "stderr": 0.0, "cost_per_test": 0.234252, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 894.128, "stderr": 0.0, "cost_per_test": 2.742624, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 901.91, "stderr": 0.0, "cost_per_test": 2.075068, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 922.502, "stderr": 0.0, "cost_per_test": 1.157097, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 1028.182, "stderr": 0.0, "cost_per_test": 3.067955, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1138.958, "stderr": 0.0, "cost_per_test": 1.100807, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1143.366, "stderr": 0.0, "cost_per_test": 3.051018, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 1198.086, "stderr": 0.0, "cost_per_test": 0.027002, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1261.026, "stderr": 0.0, "cost_per_test": 12.611383, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1282.05, "stderr": 0.0, "cost_per_test": 0.73839, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1415.18, "stderr": 0.0, "cost_per_test": 0.362556, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 1496.27, "stderr": 0.0, "cost_per_test": 1.496984, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1524.986, "stderr": 0.0, "cost_per_test": 10.04374, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 1975.746, "stderr": 0.0, "cost_per_test": 0.20359, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2112.592, "stderr": 0.0, "cost_per_test": 3.457385, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 2451.596, "stderr": 0.0, "cost_per_test": 3.296217, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 2809.288, "stderr": 0.0, "cost_per_test": 7.318404, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 2821.808, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 2879.192, "stderr": 0.0, "cost_per_test": 3.937041, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3174.424, "stderr": 0.0, "cost_per_test": 15.021633, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3182.286, "stderr": 0.0, "cost_per_test": 24.674742, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 3609.028, "stderr": 0.0, "cost_per_test": 38.53423, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 4108.97, "stderr": 0.0, "cost_per_test": 6.680874, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null } }, "data_privacy_cybersecurity": { "meta/muse_spark_1_3_max": { "accuracy": 60.0, "latency": 873.068, "stderr": 24.495, "cost_per_test": 1.819537, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 40.0, "latency": 1141.602, "stderr": 24.495, "cost_per_test": 3.424472, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 40.0, "latency": 1731.114, "stderr": 24.495, "cost_per_test": 2.741491, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 20.0, "latency": 554.804, "stderr": 20.0, "cost_per_test": 3.802232, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 20.0, "latency": 561.456, "stderr": 20.0, "cost_per_test": 0.178414, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 20.0, "latency": 641.514, "stderr": 20.0, "cost_per_test": 3.983816, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 20.0, "latency": 948.004, "stderr": 20.0, "cost_per_test": 4.57258, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 20.0, "latency": 1108.122, "stderr": 20.0, "cost_per_test": 0.161869, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 20.0, "latency": 1354.47, "stderr": 20.0, "cost_per_test": 2.842591, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 20.0, "latency": 1370.184, "stderr": 20.0, "cost_per_test": 11.031307, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 20.0, "latency": 1997.14, "stderr": 20.0, "cost_per_test": 1.406748, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 20.0, "latency": 2251.364, "stderr": 20.0, "cost_per_test": 2.805536, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-4-argon": { "accuracy": 20.0, "latency": 2394.758, "stderr": 20.0, "cost_per_test": 8.325019, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": 2679.258, "stderr": 20.0, "cost_per_test": 4.796128, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 20.0, "latency": 3410.766, "stderr": 20.0, "cost_per_test": 35.475522, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 20.0, "latency": 3526.54, "stderr": 20.0, "cost_per_test": 16.205227, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 20.0, "latency": 4005.678, "stderr": 20.0, "cost_per_test": 4.387069, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 77.594, "stderr": 0.0, "cost_per_test": 0.14917, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 78.586, "stderr": 0.0, "cost_per_test": 0.057493, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 127.518, "stderr": 0.0, "cost_per_test": 0.195281, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 159.248, "stderr": 0.0, "cost_per_test": 0.356562, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 161.092, "stderr": 0.0, "cost_per_test": 0.043571, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 205.34, "stderr": 0.0, "cost_per_test": 0.180563, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 278.552, "stderr": 0.0, "cost_per_test": 0.375904, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 298.162, "stderr": 0.0, "cost_per_test": 0.320555, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 302.728, "stderr": 0.0, "cost_per_test": 2.149328, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 329.54, "stderr": 0.0, "cost_per_test": 0.12102, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 338.276, "stderr": 0.0, "cost_per_test": 0.469152, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 344.946, "stderr": 0.0, "cost_per_test": 0.046308, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 375.346, "stderr": 0.0, "cost_per_test": 0.741361, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 379.96, "stderr": 0.0, "cost_per_test": 1.020644, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 389.142, "stderr": 0.0, "cost_per_test": 1.359549, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 407.004, "stderr": 0.0, "cost_per_test": 0.039793, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 460.386, "stderr": 0.0, "cost_per_test": 1.788615, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 463.488, "stderr": 0.0, "cost_per_test": 0.731036, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 521.026, "stderr": 0.0, "cost_per_test": 0.572555, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 535.282, "stderr": 0.0, "cost_per_test": 1.84691, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 538.888, "stderr": 0.0, "cost_per_test": 0.100374, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 560.682, "stderr": 0.0, "cost_per_test": 0.628345, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 0.0, "latency": 654.12, "stderr": 0.0, "cost_per_test": 0.83785, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 682.68, "stderr": 0.0, "cost_per_test": 2.439638, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 725.044, "stderr": 0.0, "cost_per_test": 0.400108, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 735.874, "stderr": 0.0, "cost_per_test": 0.083103, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 795.424, "stderr": 0.0, "cost_per_test": 0.204203, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 812.514, "stderr": 0.0, "cost_per_test": 2.616626, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 821.752, "stderr": 0.0, "cost_per_test": 0.067318, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 825.86, "stderr": 0.0, "cost_per_test": 3.748145, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 904.912, "stderr": 0.0, "cost_per_test": 3.103135, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 952.242, "stderr": 0.0, "cost_per_test": 1.636783, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 985.884, "stderr": 0.0, "cost_per_test": 0.697975, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1034.436, "stderr": 0.0, "cost_per_test": 3.1184, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 1096.256, "stderr": 0.0, "cost_per_test": 1.875677, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1177.988, "stderr": 0.0, "cost_per_test": 9.289774, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 1345.862, "stderr": 0.0, "cost_per_test": 3.070707, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1389.888, "stderr": 0.0, "cost_per_test": 0.392959, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1442.554, "stderr": 0.0, "cost_per_test": 0.343032, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1506.148, "stderr": 0.0, "cost_per_test": 12.882643, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1542.712, "stderr": 0.0, "cost_per_test": 2.157888, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1553.434, "stderr": 0.0, "cost_per_test": 28.026206, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1595.642, "stderr": 0.0, "cost_per_test": 20.041934, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1783.684, "stderr": 0.0, "cost_per_test": 1.156317, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 1847.046, "stderr": 0.0, "cost_per_test": 1.368981, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 1857.564, "stderr": 0.0, "cost_per_test": 9.396413, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 1871.402, "stderr": 0.0, "cost_per_test": 3.227633, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 1920.932, "stderr": 0.0, "cost_per_test": 0.178357, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 2031.03, "stderr": 0.0, "cost_per_test": 0.387754, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2115.014, "stderr": 0.0, "cost_per_test": 9.177084, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 2145.894, "stderr": 0.0, "cost_per_test": 6.673207, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 2182.8, "stderr": 0.0, "cost_per_test": 3.377518, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 2203.2, "stderr": 0.0, "cost_per_test": 0.404976, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2246.312, "stderr": 0.0, "cost_per_test": 3.575179, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 2338.224, "stderr": 0.0, "cost_per_test": 0.014473, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 2512.86, "stderr": 0.0, "cost_per_test": 5.085316, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 2722.318, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 2891.984, "stderr": 0.0, "cost_per_test": 21.050628, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 4101.838, "stderr": 0.0, "cost_per_test": 31.449565, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "emerging_companies_venture_capital": { "meta/muse_spark_1_3": { "accuracy": 40.0, "latency": 976.982, "stderr": 24.495, "cost_per_test": 1.968528, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 40.0, "latency": 1654.386, "stderr": 24.495, "cost_per_test": 1.884283, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "stepfun/step-5-preview": { "accuracy": 40.0, "latency": 1857.83, "stderr": 24.495, "cost_per_test": 0.956726, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "google/gemini-4-argon": { "accuracy": 40.0, "latency": 3078.254, "stderr": 24.495, "cost_per_test": 13.959833, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 40.0, "latency": 3218.206, "stderr": 24.495, "cost_per_test": 3.954288, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 20.0, "latency": 177.306, "stderr": 20.0, "cost_per_test": 0.051943, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.5": { "accuracy": 20.0, "latency": 659.712, "stderr": 20.0, "cost_per_test": 2.105221, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 20.0, "latency": 749.636, "stderr": 20.0, "cost_per_test": 2.002735, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 20.0, "latency": 830.042, "stderr": 20.0, "cost_per_test": 0.060188, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 20.0, "latency": 886.134, "stderr": 20.0, "cost_per_test": 0.078418, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 20.0, "latency": 1109.156, "stderr": 20.0, "cost_per_test": 3.23361, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 1407.428, "stderr": 20.0, "cost_per_test": 0.222921, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 20.0, "latency": 1442.672, "stderr": 20.0, "cost_per_test": 0.175319, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 20.0, "latency": 1499.998, "stderr": 20.0, "cost_per_test": 8.692762, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 20.0, "latency": 1607.58, "stderr": 20.0, "cost_per_test": 1.729985, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 20.0, "latency": 1722.228, "stderr": 20.0, "cost_per_test": 17.394862, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 20.0, "latency": 1784.084, "stderr": 20.0, "cost_per_test": 12.825865, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 20.0, "latency": 2170.442, "stderr": 20.0, "cost_per_test": 2.192372, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k3": { "accuracy": 20.0, "latency": 2745.84, "stderr": 20.0, "cost_per_test": 3.610617, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 20.0, "latency": 2882.79, "stderr": 20.0, "cost_per_test": 4.18071, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 20.0, "latency": 3176.12, "stderr": 20.0, "cost_per_test": 9.68991, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 20.0, "latency": 3294.328, "stderr": 20.0, "cost_per_test": 3.003809, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 20.0, "latency": 3672.84, "stderr": 20.0, "cost_per_test": 0.551792, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 61.06, "stderr": 0.0, "cost_per_test": 0.085625, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 70.562, "stderr": 0.0, "cost_per_test": 0.150699, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 160.62, "stderr": 0.0, "cost_per_test": 0.33705, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 193.59, "stderr": 0.0, "cost_per_test": 0.358329, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 207.128, "stderr": 0.0, "cost_per_test": 0.279031, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 315.706, "stderr": 0.0, "cost_per_test": 0.157036, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 322.086, "stderr": 0.0, "cost_per_test": 0.383023, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 333.714, "stderr": 0.0, "cost_per_test": 0.362288, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 356.59, "stderr": 0.0, "cost_per_test": 0.047376, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 358.07, "stderr": 0.0, "cost_per_test": 0.615337, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 398.842, "stderr": 0.0, "cost_per_test": 0.035653, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 406.88, "stderr": 0.0, "cost_per_test": 2.01759, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 415.078, "stderr": 0.0, "cost_per_test": 0.76098, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 454.6, "stderr": 0.0, "cost_per_test": 0.487189, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 481.848, "stderr": 0.0, "cost_per_test": 2.122257, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 536.078, "stderr": 0.0, "cost_per_test": 1.644529, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 573.73, "stderr": 0.0, "cost_per_test": 0.105559, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 628.536, "stderr": 0.0, "cost_per_test": 0.17285, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 662.514, "stderr": 0.0, "cost_per_test": 2.900873, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 699.088, "stderr": 0.0, "cost_per_test": 0.77175, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 780.25, "stderr": 0.0, "cost_per_test": 0.39444, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 784.232, "stderr": 0.0, "cost_per_test": 1.065606, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 0.0, "latency": 798.338, "stderr": 0.0, "cost_per_test": 0.754855, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 799.138, "stderr": 0.0, "cost_per_test": 1.664943, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 799.56, "stderr": 0.0, "cost_per_test": 2.317085, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 869.994, "stderr": 0.0, "cost_per_test": 0.309345, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 947.998, "stderr": 0.0, "cost_per_test": 3.671658, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 967.822, "stderr": 0.0, "cost_per_test": 2.63225, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 1055.05, "stderr": 0.0, "cost_per_test": 3.941127, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 1159.888, "stderr": 0.0, "cost_per_test": 1.881256, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 1165.12, "stderr": 0.0, "cost_per_test": 0.238554, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1244.242, "stderr": 0.0, "cost_per_test": 0.32382, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1276.908, "stderr": 0.0, "cost_per_test": 0.378632, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1289.842, "stderr": 0.0, "cost_per_test": 10.388772, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1316.004, "stderr": 0.0, "cost_per_test": 0.769426, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1318.76, "stderr": 0.0, "cost_per_test": 20.549812, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1326.42, "stderr": 0.0, "cost_per_test": 2.290803, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1458.474, "stderr": 0.0, "cost_per_test": 1.206641, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 1939.344, "stderr": 0.0, "cost_per_test": 1.696531, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1995.764, "stderr": 0.0, "cost_per_test": 0.990763, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 2022.664, "stderr": 0.0, "cost_per_test": 0.18381, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2237.394, "stderr": 0.0, "cost_per_test": 3.691787, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 2319.982, "stderr": 0.0, "cost_per_test": 0.026486, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.6": { "accuracy": 0.0, "latency": 2459.848, "stderr": 0.0, "cost_per_test": 4.683139, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 2558.912, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 2910.816, "stderr": 0.0, "cost_per_test": 19.621784, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2941.5, "stderr": 0.0, "cost_per_test": 13.442724, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 3143.702, "stderr": 0.0, "cost_per_test": 3.981076, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3341.218, "stderr": 0.0, "cost_per_test": 16.606099, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 3356.11, "stderr": 0.0, "cost_per_test": 21.32291, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 3656.866, "stderr": 0.0, "cost_per_test": 4.908979, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 3845.16, "stderr": 0.0, "cost_per_test": 7.559756, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 8097.042, "stderr": 0.0, "cost_per_test": 54.106639, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "employment_labor": { "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 45.712, "stderr": 0.0, "cost_per_test": 0.053854, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 80.69, "stderr": 0.0, "cost_per_test": 0.179064, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 121.512, "stderr": 0.0, "cost_per_test": 0.340745, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 121.968, "stderr": 0.0, "cost_per_test": 0.276528, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 133.024, "stderr": 0.0, "cost_per_test": 0.03689, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 216.774, "stderr": 0.0, "cost_per_test": 0.239029, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 315.964, "stderr": 0.0, "cost_per_test": 0.041572, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 327.174, "stderr": 0.0, "cost_per_test": 0.361893, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 378.896, "stderr": 0.0, "cost_per_test": 0.810734, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 389.886, "stderr": 0.0, "cost_per_test": 0.121458, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 416.436, "stderr": 0.0, "cost_per_test": 0.033177, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 421.736, "stderr": 0.0, "cost_per_test": 0.169864, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 434.874, "stderr": 0.0, "cost_per_test": 0.585035, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 449.11, "stderr": 0.0, "cost_per_test": 0.13211, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 0.0, "latency": 452.27, "stderr": 0.0, "cost_per_test": 0.589655, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 465.318, "stderr": 0.0, "cost_per_test": 0.691985, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 467.764, "stderr": 0.0, "cost_per_test": 0.514756, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 509.998, "stderr": 0.0, "cost_per_test": 2.207924, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 511.232, "stderr": 0.0, "cost_per_test": 2.671534, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 515.328, "stderr": 0.0, "cost_per_test": 0.090025, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 526.302, "stderr": 0.0, "cost_per_test": 0.921211, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 546.488, "stderr": 0.0, "cost_per_test": 0.284762, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 554.404, "stderr": 0.0, "cost_per_test": 2.284177, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 562.82, "stderr": 0.0, "cost_per_test": 0.552987, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 565.662, "stderr": 0.0, "cost_per_test": 1.185156, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 626.114, "stderr": 0.0, "cost_per_test": 0.879911, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 697.154, "stderr": 0.0, "cost_per_test": 0.018184, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 704.448, "stderr": 0.0, "cost_per_test": 2.288046, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 709.762, "stderr": 0.0, "cost_per_test": 2.404487, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 719.954, "stderr": 0.0, "cost_per_test": 1.403464, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 727.958, "stderr": 0.0, "cost_per_test": 3.168674, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 790.09, "stderr": 0.0, "cost_per_test": 1.501786, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 803.834, "stderr": 0.0, "cost_per_test": 3.000761, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 0.0, "latency": 812.294, "stderr": 0.0, "cost_per_test": 1.542385, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 865.958, "stderr": 0.0, "cost_per_test": 2.274689, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 888.802, "stderr": 0.0, "cost_per_test": 0.067578, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": 1012.946, "stderr": 0.0, "cost_per_test": 2.15024, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1018.67, "stderr": 0.0, "cost_per_test": 0.176119, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 0.0, "latency": 1039.986, "stderr": 0.0, "cost_per_test": 1.705269, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1114.998, "stderr": 0.0, "cost_per_test": 2.612081, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 1131.366, "stderr": 0.0, "cost_per_test": 1.795425, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1138.112, "stderr": 0.0, "cost_per_test": 0.205481, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1148.912, "stderr": 0.0, "cost_per_test": 1.781902, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1162.312, "stderr": 0.0, "cost_per_test": 1.29091, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1191.516, "stderr": 0.0, "cost_per_test": 1.421054, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 1204.09, "stderr": 0.0, "cost_per_test": 0.138471, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 1238.388, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1242.726, "stderr": 0.0, "cost_per_test": 0.135179, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1280.342, "stderr": 0.0, "cost_per_test": 0.735629, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1324.74, "stderr": 0.0, "cost_per_test": 7.024491, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1354.046, "stderr": 0.0, "cost_per_test": 0.293256, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 1373.15, "stderr": 0.0, "cost_per_test": 0.154545, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1375.442, "stderr": 0.0, "cost_per_test": 9.475216, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 1382.556, "stderr": 0.0, "cost_per_test": 1.421157, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1388.314, "stderr": 0.0, "cost_per_test": 16.809333, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1419.994, "stderr": 0.0, "cost_per_test": 7.485275, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1459.546, "stderr": 0.0, "cost_per_test": 14.366748, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 1577.85, "stderr": 0.0, "cost_per_test": 0.045526, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1699.206, "stderr": 0.0, "cost_per_test": 0.881184, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 1712.742, "stderr": 0.0, "cost_per_test": 2.622256, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 1928.642, "stderr": 0.0, "cost_per_test": 7.141814, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1978.914, "stderr": 0.0, "cost_per_test": 0.989703, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 1986.27, "stderr": 0.0, "cost_per_test": 2.240265, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 2165.544, "stderr": 0.0, "cost_per_test": 6.38748, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2217.096, "stderr": 0.0, "cost_per_test": 3.677382, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2260.078, "stderr": 0.0, "cost_per_test": 2.442027, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 0.0, "latency": 2386.33, "stderr": 0.0, "cost_per_test": 3.463162, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2613.692, "stderr": 0.0, "cost_per_test": 10.060231, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 2695.91, "stderr": 0.0, "cost_per_test": 1.709298, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 2860.095, "stderr": 0.0, "cost_per_test": 14.514568, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 2955.226, "stderr": 0.0, "cost_per_test": 18.043089, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 2960.05, "stderr": 0.0, "cost_per_test": 5.072256, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 3234.68, "stderr": 0.0, "cost_per_test": 0.382305, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 3521.574, "stderr": 0.0, "cost_per_test": 28.089397, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3527.114, "stderr": 0.0, "cost_per_test": 17.260921, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 4102.402, "stderr": 0.0, "cost_per_test": 6.850325, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null } }, "energy_natural_resources": { "google/gemini-3.7-flash": { "accuracy": 40.0, "latency": 447.692, "stderr": 24.495, "cost_per_test": 3.885089, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 40.0, "latency": 928.796, "stderr": 24.495, "cost_per_test": 6.265695, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 40.0, "latency": 1538.616, "stderr": 24.495, "cost_per_test": 0.224328, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 40.0, "latency": 1654.116, "stderr": 24.495, "cost_per_test": 3.699196, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 40.0, "latency": 1767.97, "stderr": 24.495, "cost_per_test": 1.160357, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 40.0, "latency": 2031.21, "stderr": 24.495, "cost_per_test": 3.505056, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 40.0, "latency": 2061.922, "stderr": 24.495, "cost_per_test": 4.142205, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 20.0, "latency": 388.468, "stderr": 20.0, "cost_per_test": 0.983444, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 20.0, "latency": 398.398, "stderr": 20.0, "cost_per_test": 0.219636, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.5": { "accuracy": 20.0, "latency": 685.118, "stderr": 20.0, "cost_per_test": 3.669584, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 20.0, "latency": 753.42, "stderr": 20.0, "cost_per_test": 0.613118, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 20.0, "latency": 853.212, "stderr": 20.0, "cost_per_test": 2.004243, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 20.0, "latency": 907.39, "stderr": 20.0, "cost_per_test": 3.111032, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 20.0, "latency": 965.426, "stderr": 20.0, "cost_per_test": 6.052018, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 20.0, "latency": 1031.512, "stderr": 20.0, "cost_per_test": 1.224794, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 20.0, "latency": 1183.832, "stderr": 20.0, "cost_per_test": 0.100514, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 20.0, "latency": 1207.448, "stderr": 20.0, "cost_per_test": 0.670627, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 20.0, "latency": 1396.084, "stderr": 20.0, "cost_per_test": 4.475508, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 20.0, "latency": 1405.156, "stderr": 20.0, "cost_per_test": 9.063448, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.5": { "accuracy": 20.0, "latency": 1415.76, "stderr": 20.0, "cost_per_test": 11.82685, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 20.0, "latency": 1560.884, "stderr": 20.0, "cost_per_test": 14.535287, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 20.0, "latency": 1638.978, "stderr": 20.0, "cost_per_test": 7.731016, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 20.0, "latency": 1700.96, "stderr": 20.0, "cost_per_test": 1.80131, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 20.0, "latency": 1722.806, "stderr": 20.0, "cost_per_test": 17.774066, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 20.0, "latency": 1738.252, "stderr": 20.0, "cost_per_test": 17.557582, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 1774.802, "stderr": 20.0, "cost_per_test": 0.288516, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 20.0, "latency": 1808.578, "stderr": 20.0, "cost_per_test": 32.1783, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 20.0, "latency": 1988.198, "stderr": 20.0, "cost_per_test": 10.743255, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": 1996.308, "stderr": 20.0, "cost_per_test": 5.233643, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 20.0, "latency": 2085.852, "stderr": 20.0, "cost_per_test": 13.632671, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 20.0, "latency": 2531.316, "stderr": 20.0, "cost_per_test": 3.273878, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 20.0, "latency": 2822.94, "stderr": 20.0, "cost_per_test": 5.852595, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/ember-1": { "accuracy": 20.0, "latency": 2907.838, "stderr": 20.0, "cost_per_test": 4.82694, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 20.0, "latency": 2954.94, "stderr": 20.0, "cost_per_test": 0.562571, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 20.0, "latency": 3028.416, "stderr": 20.0, "cost_per_test": 14.055481, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 20.0, "latency": 3374.346, "stderr": 20.0, "cost_per_test": 4.405884, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 20.0, "latency": 3653.828, "stderr": 20.0, "cost_per_test": 26.383042, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 20.0, "latency": 3747.78, "stderr": 20.0, "cost_per_test": 31.649398, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 87.666, "stderr": 0.0, "cost_per_test": 0.205085, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 93.152, "stderr": 0.0, "cost_per_test": 0.235869, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 161.315, "stderr": 0.0, "cost_per_test": 0.058036, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 204.76, "stderr": 0.0, "cost_per_test": 0.350581, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 327.076, "stderr": 0.0, "cost_per_test": 0.37291, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 340.014, "stderr": 0.0, "cost_per_test": 0.495202, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 350.94, "stderr": 0.0, "cost_per_test": 1.916498, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 360.398, "stderr": 0.0, "cost_per_test": 0.330094, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 368.87, "stderr": 0.0, "cost_per_test": 0.181632, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 382.348, "stderr": 0.0, "cost_per_test": 0.164729, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 384.886, "stderr": 0.0, "cost_per_test": 0.461391, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 411.296, "stderr": 0.0, "cost_per_test": 0.138008, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 411.552, "stderr": 0.0, "cost_per_test": 0.399543, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 413.794, "stderr": 0.0, "cost_per_test": 0.042392, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 458.446, "stderr": 0.0, "cost_per_test": 2.006452, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 506.004, "stderr": 0.0, "cost_per_test": 0.05328, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 512.687, "stderr": 0.0, "cost_per_test": 0.30825, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 601.502, "stderr": 0.0, "cost_per_test": 0.939334, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 608.292, "stderr": 0.0, "cost_per_test": 0.891986, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 815.562, "stderr": 0.0, "cost_per_test": 1.042261, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 953.146, "stderr": 0.0, "cost_per_test": 0.646701, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 1088.188, "stderr": 0.0, "cost_per_test": 9.258809, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 1167.466, "stderr": 0.0, "cost_per_test": 0.032069, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 1169.822, "stderr": 0.0, "cost_per_test": 8.24435, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 1203.204, "stderr": 0.0, "cost_per_test": 1.814778, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 1283.055, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 1391.76, "stderr": 0.0, "cost_per_test": 0.539658, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1433.85, "stderr": 0.0, "cost_per_test": 0.330464, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 1484.988, "stderr": 0.0, "cost_per_test": 1.559743, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 1540.416, "stderr": 0.0, "cost_per_test": 0.125628, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 1571.69, "stderr": 0.0, "cost_per_test": 4.843052, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1891.152, "stderr": 0.0, "cost_per_test": 1.207559, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1943.202, "stderr": 0.0, "cost_per_test": 50.660049, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 1976.13, "stderr": 0.0, "cost_per_test": 3.629968, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 2561.954, "stderr": 0.0, "cost_per_test": 0.226112, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3233.776, "stderr": 0.0, "cost_per_test": 17.609237, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 4775.52, "stderr": 0.0, "cost_per_test": 9.063118, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 7358.538, "stderr": 0.0, "cost_per_test": 54.268239, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "environmental_esg": { "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 59.852, "stderr": 0.0, "cost_per_test": 0.072209, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 86.018, "stderr": 0.0, "cost_per_test": 0.222426, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 136.838, "stderr": 0.0, "cost_per_test": 0.279814, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 154.234, "stderr": 0.0, "cost_per_test": 0.230865, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 157.04, "stderr": 0.0, "cost_per_test": 0.040915, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 223.294, "stderr": 0.0, "cost_per_test": 0.526047, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 304.116, "stderr": 0.0, "cost_per_test": 0.36933, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 313.084, "stderr": 0.0, "cost_per_test": 0.22878, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 375.606, "stderr": 0.0, "cost_per_test": 0.483135, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 412.29, "stderr": 0.0, "cost_per_test": 0.64771, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 421.46, "stderr": 0.0, "cost_per_test": 0.603349, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 453.494, "stderr": 0.0, "cost_per_test": 0.562732, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 470.746, "stderr": 0.0, "cost_per_test": 2.036146, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 484.558, "stderr": 0.0, "cost_per_test": 0.058194, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 0.0, "latency": 496.367, "stderr": 0.0, "cost_per_test": 0.594901, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 510.966, "stderr": 0.0, "cost_per_test": 0.033992, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 600.832, "stderr": 0.0, "cost_per_test": 0.102528, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 612.728, "stderr": 0.0, "cost_per_test": 0.620051, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 664.552, "stderr": 0.0, "cost_per_test": 3.055555, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 666.612, "stderr": 0.0, "cost_per_test": 3.428704, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 677.67, "stderr": 0.0, "cost_per_test": 1.19933, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 698.84, "stderr": 0.0, "cost_per_test": 0.167209, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 702.338, "stderr": 0.0, "cost_per_test": 1.074487, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 723.794, "stderr": 0.0, "cost_per_test": 0.048721, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 746.228, "stderr": 0.0, "cost_per_test": 0.310354, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 762.87, "stderr": 0.0, "cost_per_test": 3.081603, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 765.464, "stderr": 0.0, "cost_per_test": 2.260337, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 843.6, "stderr": 0.0, "cost_per_test": 2.449822, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 882.072, "stderr": 0.0, "cost_per_test": 2.235795, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 0.0, "latency": 891.213, "stderr": 0.0, "cost_per_test": 1.983442, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 903.288, "stderr": 0.0, "cost_per_test": 0.086836, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 942.3, "stderr": 0.0, "cost_per_test": 3.664358, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": 987.19, "stderr": 0.0, "cost_per_test": 1.971553, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 1009.74, "stderr": 0.0, "cost_per_test": 1.92219, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 1030.806, "stderr": 0.0, "cost_per_test": 2.99183, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1066.538, "stderr": 0.0, "cost_per_test": 0.207105, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1111.074, "stderr": 0.0, "cost_per_test": 7.575723, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1138.234, "stderr": 0.0, "cost_per_test": 3.211685, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 1173.572, "stderr": 0.0, "cost_per_test": 0.395489, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 1195.644, "stderr": 0.0, "cost_per_test": 3.029146, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1250.098, "stderr": 0.0, "cost_per_test": 0.232534, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 1322.42, "stderr": 0.0, "cost_per_test": 0.248012, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1397.79, "stderr": 0.0, "cost_per_test": 1.189505, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1402.704, "stderr": 0.0, "cost_per_test": 0.30468, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1423.762, "stderr": 0.0, "cost_per_test": 1.886054, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1434.172, "stderr": 0.0, "cost_per_test": 17.289629, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 0.0, "latency": 1476.363, "stderr": 0.0, "cost_per_test": 1.666729, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 1534.666, "stderr": 0.0, "cost_per_test": 0.173828, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 1606.52, "stderr": 0.0, "cost_per_test": 3.219091, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1679.81, "stderr": 0.0, "cost_per_test": 17.039801, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1687.488, "stderr": 0.0, "cost_per_test": 10.410854, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1709.262, "stderr": 0.0, "cost_per_test": 0.994698, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1732.44, "stderr": 0.0, "cost_per_test": 1.768355, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1796.954, "stderr": 0.0, "cost_per_test": 0.168238, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1871.692, "stderr": 0.0, "cost_per_test": 0.897665, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 1902.6, "stderr": 0.0, "cost_per_test": 1.529383, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1922.718, "stderr": 0.0, "cost_per_test": 0.97515, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1935.266, "stderr": 0.0, "cost_per_test": 12.224012, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 2090.494, "stderr": 0.0, "cost_per_test": 2.2076, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 2321.664, "stderr": 0.0, "cost_per_test": 8.278315, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2387.788, "stderr": 0.0, "cost_per_test": 2.678325, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2481.084, "stderr": 0.0, "cost_per_test": 4.101782, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 2493.466, "stderr": 0.0, "cost_per_test": 2.821772, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 2747.75, "stderr": 0.0, "cost_per_test": 0.02734, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 2749.164, "stderr": 0.0, "cost_per_test": 3.563589, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 2788.346, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 2805.428, "stderr": 0.0, "cost_per_test": 6.10019, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 2817.198, "stderr": 0.0, "cost_per_test": 8.754557, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2862.154, "stderr": 0.0, "cost_per_test": 11.199341, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 3334.934, "stderr": 0.0, "cost_per_test": 4.093269, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3502.29, "stderr": 0.0, "cost_per_test": 24.257857, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 3980.458, "stderr": 0.0, "cost_per_test": 0.493606, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 4047.096, "stderr": 0.0, "cost_per_test": 18.079139, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 4134.27, "stderr": 0.0, "cost_per_test": 24.419754, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 0.0, "latency": 4334.738, "stderr": 0.0, "cost_per_test": 3.914912, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 9176.072, "stderr": 0.0, "cost_per_test": 54.111114, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "funds_asset_management": { "meta/muse_spark_1_1": { "accuracy": 40.0, "latency": 1163.572, "stderr": 24.495, "cost_per_test": 0.99674, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 40.0, "latency": 2412.33, "stderr": 24.495, "cost_per_test": 10.601161, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 20.0, "latency": 676.124, "stderr": 20.0, "cost_per_test": 2.138169, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 20.0, "latency": 950.384, "stderr": 20.0, "cost_per_test": 2.607789, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 20.0, "latency": 985.742, "stderr": 20.0, "cost_per_test": 0.077692, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 20.0, "latency": 1059.502, "stderr": 20.0, "cost_per_test": 3.179398, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 20.0, "latency": 1160.458, "stderr": 20.0, "cost_per_test": 5.475664, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 20.0, "latency": 1393.648, "stderr": 20.0, "cost_per_test": 0.106311, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 20.0, "latency": 1654.824, "stderr": 20.0, "cost_per_test": 11.982318, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 20.0, "latency": 2280.372, "stderr": 20.0, "cost_per_test": 1.20772, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 2292.694, "stderr": 20.0, "cost_per_test": 0.275778, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 20.0, "latency": 2651.038, "stderr": 20.0, "cost_per_test": 5.381658, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.7": { "accuracy": 20.0, "latency": 2684.486, "stderr": 20.0, "cost_per_test": 12.823095, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.6": { "accuracy": 20.0, "latency": 3359.162, "stderr": 20.0, "cost_per_test": 4.043474, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/ember-1": { "accuracy": 20.0, "latency": 3438.736, "stderr": 20.0, "cost_per_test": 4.870552, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 20.0, "latency": 4279.908, "stderr": 20.0, "cost_per_test": 3.138564, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 20.0, "latency": 5249.848, "stderr": 20.0, "cost_per_test": 43.038067, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 81.95, "stderr": 0.0, "cost_per_test": 0.142138, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 213.786, "stderr": 0.0, "cost_per_test": 0.705274, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 245.102, "stderr": 0.0, "cost_per_test": 0.620247, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 246.674, "stderr": 0.0, "cost_per_test": 0.061406, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 267.027, "stderr": 0.0, "cost_per_test": 0.116707, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 309.586, "stderr": 0.0, "cost_per_test": 0.313043, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 322.28, "stderr": 0.0, "cost_per_test": 0.253859, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 325.206, "stderr": 0.0, "cost_per_test": 0.438451, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 344.172, "stderr": 0.0, "cost_per_test": 0.132025, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 400.146, "stderr": 0.0, "cost_per_test": 2.291932, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 404.806, "stderr": 0.0, "cost_per_test": 0.057115, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 460.218, "stderr": 0.0, "cost_per_test": 0.716192, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 512.676, "stderr": 0.0, "cost_per_test": 0.040011, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 522.94, "stderr": 0.0, "cost_per_test": 2.30202, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 523.878, "stderr": 0.0, "cost_per_test": 0.955566, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 549.724, "stderr": 0.0, "cost_per_test": 0.736004, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 569.218, "stderr": 0.0, "cost_per_test": 2.973017, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 607.44, "stderr": 0.0, "cost_per_test": 0.113815, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 644.558, "stderr": 0.0, "cost_per_test": 0.636171, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 683.726, "stderr": 0.0, "cost_per_test": 0.718616, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 754.854, "stderr": 0.0, "cost_per_test": 1.259469, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 774.918, "stderr": 0.0, "cost_per_test": 0.499947, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 828.736, "stderr": 0.0, "cost_per_test": 0.190615, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 884.656, "stderr": 0.0, "cost_per_test": 3.027072, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 894.614, "stderr": 0.0, "cost_per_test": 3.568526, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 963.842, "stderr": 0.0, "cost_per_test": 6.318696, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 974.284, "stderr": 0.0, "cost_per_test": 3.406867, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 1081.948, "stderr": 0.0, "cost_per_test": 4.58965, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1152.156, "stderr": 0.0, "cost_per_test": 3.25694, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 1172.158, "stderr": 0.0, "cost_per_test": 2.077847, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 1206.37, "stderr": 0.0, "cost_per_test": 5.020577, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 1298.726, "stderr": 0.0, "cost_per_test": 3.647174, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 1338.436, "stderr": 0.0, "cost_per_test": 0.260312, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1369.844, "stderr": 0.0, "cost_per_test": 1.427877, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1389.99, "stderr": 0.0, "cost_per_test": 2.133545, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1505.88, "stderr": 0.0, "cost_per_test": 0.351581, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1624.184, "stderr": 0.0, "cost_per_test": 1.845597, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1677.146, "stderr": 0.0, "cost_per_test": 0.219769, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1687.568, "stderr": 0.0, "cost_per_test": 12.061978, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1783.184, "stderr": 0.0, "cost_per_test": 0.392274, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1785.652, "stderr": 0.0, "cost_per_test": 1.192459, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 0.0, "latency": 1831.468, "stderr": 0.0, "cost_per_test": 2.411987, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1928.168, "stderr": 0.0, "cost_per_test": 35.059835, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 2127.48, "stderr": 0.0, "cost_per_test": 1.258092, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 2129.294, "stderr": 0.0, "cost_per_test": 0.202207, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 2223.198, "stderr": 0.0, "cost_per_test": 32.834156, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2262.452, "stderr": 0.0, "cost_per_test": 4.211796, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 2407.934, "stderr": 0.0, "cost_per_test": 10.393235, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 2408.748, "stderr": 0.0, "cost_per_test": 1.625214, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 2922.548, "stderr": 0.0, "cost_per_test": 0.051392, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 3107.416, "stderr": 0.0, "cost_per_test": 13.033612, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 3112.73, "stderr": 0.0, "cost_per_test": 5.130276, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 3159.604, "stderr": 0.0, "cost_per_test": 4.299795, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3329.837, "stderr": 0.0, "cost_per_test": 21.45996, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 3375.11, "stderr": 0.0, "cost_per_test": 6.966914, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 3576.656, "stderr": 0.0, "cost_per_test": 0.771439, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 3845.516, "stderr": 0.0, "cost_per_test": 28.228604, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 3933.64, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 4008.234, "stderr": 0.0, "cost_per_test": 19.376575, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "healthcare_life_sciences": { "meta/muse_spark_1_2": { "accuracy": 60.0, "latency": 1815.46, "stderr": 24.495, "cost_per_test": 2.602242, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 60.0, "latency": 2623.2, "stderr": 24.495, "cost_per_test": 4.871642, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.5": { "accuracy": 40.0, "latency": 761.65, "stderr": 24.495, "cost_per_test": 3.027345, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 40.0, "latency": 909.876, "stderr": 24.495, "cost_per_test": 1.865298, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 40.0, "latency": 1043.858, "stderr": 24.495, "cost_per_test": 0.056365, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 40.0, "latency": 1130.752, "stderr": 24.495, "cost_per_test": 2.30716, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 40.0, "latency": 1343.57, "stderr": 24.495, "cost_per_test": 10.479166, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 40.0, "latency": 2000.958, "stderr": 24.495, "cost_per_test": 2.838311, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 20.0, "latency": 511.32, "stderr": 20.0, "cost_per_test": 3.002272, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 20.0, "latency": 541.786, "stderr": 20.0, "cost_per_test": 0.121036, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 20.0, "latency": 554.702, "stderr": 20.0, "cost_per_test": 3.175621, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 20.0, "latency": 622.19, "stderr": 20.0, "cost_per_test": 0.166133, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 20.0, "latency": 833.11, "stderr": 20.0, "cost_per_test": 3.474289, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 20.0, "latency": 873.216, "stderr": 20.0, "cost_per_test": 2.706824, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 20.0, "latency": 1001.374, "stderr": 20.0, "cost_per_test": 0.991405, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 20.0, "latency": 1207.572, "stderr": 20.0, "cost_per_test": 0.164353, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 20.0, "latency": 1277.572, "stderr": 20.0, "cost_per_test": 3.819729, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 20.0, "latency": 1360.682, "stderr": 20.0, "cost_per_test": 2.190988, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 20.0, "latency": 1436.112, "stderr": 20.0, "cost_per_test": 1.4604, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 20.0, "latency": 1607.468, "stderr": 20.0, "cost_per_test": 0.09248, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 20.0, "latency": 1645.034, "stderr": 20.0, "cost_per_test": 23.076834, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 20.0, "latency": 2020.436, "stderr": 20.0, "cost_per_test": 1.267174, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 20.0, "latency": 2182.822, "stderr": 20.0, "cost_per_test": 1.715622, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 2184.576, "stderr": 20.0, "cost_per_test": 0.21495, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": 2196.24, "stderr": 20.0, "cost_per_test": 4.04281, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 20.0, "latency": 2200.132, "stderr": 20.0, "cost_per_test": 1.289815, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "grok/grok-4.6": { "accuracy": 20.0, "latency": 2323.636, "stderr": 20.0, "cost_per_test": 4.837427, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 20.0, "latency": 2340.06, "stderr": 20.0, "cost_per_test": 8.616817, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 20.0, "latency": 2753.772, "stderr": 20.0, "cost_per_test": 14.517611, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 20.0, "latency": 2917.028, "stderr": 20.0, "cost_per_test": 8.843854, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 20.0, "latency": 2998.624, "stderr": 20.0, "cost_per_test": 2.223213, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 20.0, "latency": 3583.414, "stderr": 20.0, "cost_per_test": 24.921853, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 20.0, "latency": 4054.91, "stderr": 20.0, "cost_per_test": 28.864668, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 46.904, "stderr": 0.0, "cost_per_test": 0.049241, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 112.726, "stderr": 0.0, "cost_per_test": 0.210725, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 141.404, "stderr": 0.0, "cost_per_test": 0.284177, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 172.172, "stderr": 0.0, "cost_per_test": 0.43244, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 189.678, "stderr": 0.0, "cost_per_test": 0.047604, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 215.342, "stderr": 0.0, "cost_per_test": 0.288454, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 377.358, "stderr": 0.0, "cost_per_test": 0.050679, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 386.764, "stderr": 0.0, "cost_per_test": 0.191612, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 435.376, "stderr": 0.0, "cost_per_test": 0.848233, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 444.818, "stderr": 0.0, "cost_per_test": 0.043026, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 467.984, "stderr": 0.0, "cost_per_test": 0.463686, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 485.53, "stderr": 0.0, "cost_per_test": 0.860793, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 496.698, "stderr": 0.0, "cost_per_test": 0.183866, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 511.986, "stderr": 0.0, "cost_per_test": 0.572801, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 575.66, "stderr": 0.0, "cost_per_test": 2.41183, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 597.642, "stderr": 0.0, "cost_per_test": 0.741325, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 617.856, "stderr": 0.0, "cost_per_test": 0.759264, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 691.188, "stderr": 0.0, "cost_per_test": 1.096454, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 702.102, "stderr": 0.0, "cost_per_test": 0.289523, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 772.625, "stderr": 0.0, "cost_per_test": 1.351574, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 802.494, "stderr": 0.0, "cost_per_test": 2.391775, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 812.896, "stderr": 0.0, "cost_per_test": 4.010513, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 934.792, "stderr": 0.0, "cost_per_test": 2.130891, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 965.994, "stderr": 0.0, "cost_per_test": 3.200117, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 997.662, "stderr": 0.0, "cost_per_test": 0.239122, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 1021.66, "stderr": 0.0, "cost_per_test": 3.327293, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 1131.514, "stderr": 0.0, "cost_per_test": 1.902096, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 1426.678, "stderr": 0.0, "cost_per_test": 0.336544, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 1443.008, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1477.564, "stderr": 0.0, "cost_per_test": 1.476459, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1509.924, "stderr": 0.0, "cost_per_test": 20.8409, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1594.302, "stderr": 0.0, "cost_per_test": 11.601398, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1759.028, "stderr": 0.0, "cost_per_test": 0.66545, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1954.012, "stderr": 0.0, "cost_per_test": 14.122835, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 2007.388, "stderr": 0.0, "cost_per_test": 0.460233, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 2024.916, "stderr": 0.0, "cost_per_test": 0.171673, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2395.058, "stderr": 0.0, "cost_per_test": 3.397341, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 2823.914, "stderr": 0.0, "cost_per_test": 0.027595, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 3250.102, "stderr": 0.0, "cost_per_test": 0.762902, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 3718.032, "stderr": 0.0, "cost_per_test": 4.342941, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 4075.734, "stderr": 0.0, "cost_per_test": 19.894952, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 4427.734, "stderr": 0.0, "cost_per_test": 41.981832, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 4695.758, "stderr": 0.0, "cost_per_test": 6.264992, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null } }, "immigration": { "meta/muse_spark_1_3_max": { "accuracy": 40.0, "latency": 846.297, "stderr": 24.495, "cost_per_test": 1.891702, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 40.0, "latency": 1287.105, "stderr": 24.495, "cost_per_test": 1.326868, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 20.0, "latency": 529.208, "stderr": 20.0, "cost_per_test": 0.483627, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 20.0, "latency": 1803.852, "stderr": 20.0, "cost_per_test": 2.10625, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 20.0, "latency": 2775.568, "stderr": 20.0, "cost_per_test": 9.272635, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 55.438, "stderr": 0.0, "cost_per_test": 0.055276, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 77.046, "stderr": 0.0, "cost_per_test": 0.149835, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 121.822, "stderr": 0.0, "cost_per_test": 0.154995, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 126.21, "stderr": 0.0, "cost_per_test": 0.037666, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 148.65, "stderr": 0.0, "cost_per_test": 0.35425, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 232.172, "stderr": 0.0, "cost_per_test": 0.012559, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 240.314, "stderr": 0.0, "cost_per_test": 0.846698, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 298.584, "stderr": 0.0, "cost_per_test": 0.74639, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 309.228, "stderr": 0.0, "cost_per_test": 0.127284, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 328.71, "stderr": 0.0, "cost_per_test": 0.042242, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 333.842, "stderr": 0.0, "cost_per_test": 0.519471, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 341.998, "stderr": 0.0, "cost_per_test": 0.36647, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 348.37, "stderr": 0.0, "cost_per_test": 2.310037, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 419.566, "stderr": 0.0, "cost_per_test": 1.560451, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 431.31, "stderr": 0.0, "cost_per_test": 0.284081, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 444.094, "stderr": 0.0, "cost_per_test": 0.391182, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 456.404, "stderr": 0.0, "cost_per_test": 0.028647, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 513.374, "stderr": 0.0, "cost_per_test": 0.762367, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 537.584, "stderr": 0.0, "cost_per_test": 0.513525, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 539.64, "stderr": 0.0, "cost_per_test": 1.422373, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 571.14, "stderr": 0.0, "cost_per_test": 0.151627, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 575.12, "stderr": 0.0, "cost_per_test": 0.227351, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 589.338, "stderr": 0.0, "cost_per_test": 0.152164, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 592.33, "stderr": 0.0, "cost_per_test": 2.608087, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 616.432, "stderr": 0.0, "cost_per_test": 1.548993, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 630.912, "stderr": 0.0, "cost_per_test": 0.110658, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 634.518, "stderr": 0.0, "cost_per_test": 0.083903, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 637.036, "stderr": 0.0, "cost_per_test": 0.459943, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 666.162, "stderr": 0.0, "cost_per_test": 0.041542, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 682.574, "stderr": 0.0, "cost_per_test": 2.985935, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 694.87, "stderr": 0.0, "cost_per_test": 1.962883, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 707.206, "stderr": 0.0, "cost_per_test": 0.851971, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 757.904, "stderr": 0.0, "cost_per_test": 2.686527, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 770.674, "stderr": 0.0, "cost_per_test": 2.857841, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 771.262, "stderr": 0.0, "cost_per_test": 2.396939, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 783.178, "stderr": 0.0, "cost_per_test": 1.301215, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 844.856, "stderr": 0.0, "cost_per_test": 1.515755, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": 859.238, "stderr": 0.0, "cost_per_test": 1.580757, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 985.112, "stderr": 0.0, "cost_per_test": 0.069274, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1017.402, "stderr": 0.0, "cost_per_test": 5.823174, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1046.02, "stderr": 0.0, "cost_per_test": 2.059771, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1062.394, "stderr": 0.0, "cost_per_test": 0.987413, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1119.492, "stderr": 0.0, "cost_per_test": 0.128623, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1172.736, "stderr": 0.0, "cost_per_test": 11.161235, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1322.102, "stderr": 0.0, "cost_per_test": 0.650266, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1326.436, "stderr": 0.0, "cost_per_test": 1.8212, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1347.378, "stderr": 0.0, "cost_per_test": 7.846506, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1442.084, "stderr": 0.0, "cost_per_test": 0.342586, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1456.41, "stderr": 0.0, "cost_per_test": 8.05746, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1489.498, "stderr": 0.0, "cost_per_test": 14.198276, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 1555.058, "stderr": 0.0, "cost_per_test": 0.160872, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1558.018, "stderr": 0.0, "cost_per_test": 0.755618, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 1687.584, "stderr": 0.0, "cost_per_test": 1.581573, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 1798.635, "stderr": 0.0, "cost_per_test": 0.195853, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1834.206, "stderr": 0.0, "cost_per_test": 0.79121, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 1971.066, "stderr": 0.0, "cost_per_test": 8.081965, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2101.604, "stderr": 0.0, "cost_per_test": 2.214603, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2130.83, "stderr": 0.0, "cost_per_test": 2.704387, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 2152.852, "stderr": 0.0, "cost_per_test": 6.833464, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 2243.912, "stderr": 0.0, "cost_per_test": 1.390074, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 2299.654, "stderr": 0.0, "cost_per_test": 3.068155, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 2453.73, "stderr": 0.0, "cost_per_test": 0.399209, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2574.232, "stderr": 0.0, "cost_per_test": 9.207192, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 2833.772, "stderr": 0.0, "cost_per_test": 2.00223, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 2903.422, "stderr": 0.0, "cost_per_test": 18.52264, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 2972.516, "stderr": 0.0, "cost_per_test": 0.178109, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 2995.124, "stderr": 0.0, "cost_per_test": 19.864027, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 3031.804, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 3189.722, "stderr": 0.0, "cost_per_test": 25.924114, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3260.778, "stderr": 0.0, "cost_per_test": 13.84088, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 0.0, "latency": 3374.932, "stderr": 0.0, "cost_per_test": 3.07805, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } }, "insurance": { "google/gemini-3.7-flash": { "accuracy": 20.0, "latency": 413.094, "stderr": 20.0, "cost_per_test": 2.436289, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 73.878, "stderr": 0.0, "cost_per_test": 0.087874, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 103.32, "stderr": 0.0, "cost_per_test": 0.182741, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 132.178, "stderr": 0.0, "cost_per_test": 0.044849, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 137.038, "stderr": 0.0, "cost_per_test": 0.213394, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 145.796, "stderr": 0.0, "cost_per_test": 0.345132, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 257.758, "stderr": 0.0, "cost_per_test": 0.666001, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 283.512, "stderr": 0.0, "cost_per_test": 0.376663, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 353.55, "stderr": 0.0, "cost_per_test": 0.247595, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 358.856, "stderr": 0.0, "cost_per_test": 0.368749, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 359.782, "stderr": 0.0, "cost_per_test": 0.050634, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 401.878, "stderr": 0.0, "cost_per_test": 1.769687, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 405.378, "stderr": 0.0, "cost_per_test": 0.628366, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 412.404, "stderr": 0.0, "cost_per_test": 0.441831, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 467.072, "stderr": 0.0, "cost_per_test": 0.033672, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 473.472, "stderr": 0.0, "cost_per_test": 0.555887, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 0.0, "latency": 491.63, "stderr": 0.0, "cost_per_test": 0.58862, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 509.878, "stderr": 0.0, "cost_per_test": 0.089394, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 526.822, "stderr": 0.0, "cost_per_test": 0.601521, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 529.27, "stderr": 0.0, "cost_per_test": 2.685643, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 555.876, "stderr": 0.0, "cost_per_test": 0.801823, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 607.276, "stderr": 0.0, "cost_per_test": 0.042314, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 611.662, "stderr": 0.0, "cost_per_test": 0.866939, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 631.634, "stderr": 0.0, "cost_per_test": 2.417604, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 661.584, "stderr": 0.0, "cost_per_test": 2.129824, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 680.89, "stderr": 0.0, "cost_per_test": 0.307965, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 708.134, "stderr": 0.0, "cost_per_test": 1.663014, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 721.662, "stderr": 0.0, "cost_per_test": 0.060043, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 757.324, "stderr": 0.0, "cost_per_test": 1.374128, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 762.124, "stderr": 0.0, "cost_per_test": 2.10689, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 802.086, "stderr": 0.0, "cost_per_test": 2.401502, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 824.586, "stderr": 0.0, "cost_per_test": 0.16401, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 0.0, "latency": 845.222, "stderr": 0.0, "cost_per_test": 1.773661, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 848.144, "stderr": 0.0, "cost_per_test": 0.151326, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 863.304, "stderr": 0.0, "cost_per_test": 0.168232, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 879.704, "stderr": 0.0, "cost_per_test": 0.471893, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 899.01, "stderr": 0.0, "cost_per_test": 2.265639, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": 967.18, "stderr": 0.0, "cost_per_test": 2.204869, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 986.986, "stderr": 0.0, "cost_per_test": 0.11707, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 999.666, "stderr": 0.0, "cost_per_test": 2.759668, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 1077.368, "stderr": 0.0, "cost_per_test": 3.594349, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 0.0, "latency": 1085.91, "stderr": 0.0, "cost_per_test": 1.256553, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1140.7, "stderr": 0.0, "cost_per_test": 1.138776, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1205.068, "stderr": 0.0, "cost_per_test": 1.05308, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 1209.736, "stderr": 0.0, "cost_per_test": 1.949242, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1239.256, "stderr": 0.0, "cost_per_test": 8.5933, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1278.736, "stderr": 0.0, "cost_per_test": 9.836148, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1287.198, "stderr": 0.0, "cost_per_test": 1.917632, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1289.65, "stderr": 0.0, "cost_per_test": 15.655481, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 1296.19, "stderr": 0.0, "cost_per_test": 0.286257, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1350.64, "stderr": 0.0, "cost_per_test": 0.819488, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1417.02, "stderr": 0.0, "cost_per_test": 15.611186, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1460.754, "stderr": 0.0, "cost_per_test": 9.118767, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 1502.57, "stderr": 0.0, "cost_per_test": 1.707085, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1504.748, "stderr": 0.0, "cost_per_test": 0.879072, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1590.182, "stderr": 0.0, "cost_per_test": 0.269577, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 1719.334, "stderr": 0.0, "cost_per_test": 0.008422, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1827.906, "stderr": 0.0, "cost_per_test": 1.08578, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 1881.516, "stderr": 0.0, "cost_per_test": 2.905753, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 1949.21, "stderr": 0.0, "cost_per_test": 6.035978, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 1985.446, "stderr": 0.0, "cost_per_test": 0.187751, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 2064.252, "stderr": 0.0, "cost_per_test": 1.401873, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2146.625, "stderr": 0.0, "cost_per_test": 2.868148, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 2212.132, "stderr": 0.0, "cost_per_test": 2.324792, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 2233.016, "stderr": 0.0, "cost_per_test": 3.64804, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2425.114, "stderr": 0.0, "cost_per_test": 2.559806, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2443.796, "stderr": 0.0, "cost_per_test": 10.158566, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 2551.164, "stderr": 0.0, "cost_per_test": 4.703644, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 2860.728, "stderr": 0.0, "cost_per_test": 16.375474, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 2865.77, "stderr": 0.0, "cost_per_test": 7.165675, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 3236.886, "stderr": 0.0, "cost_per_test": 0.493207, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 3435.596, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3460.936, "stderr": 0.0, "cost_per_test": 15.744184, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 3746.53, "stderr": 0.0, "cost_per_test": 22.385777, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.6": { "accuracy": 0.0, "latency": 3828.232, "stderr": 0.0, "cost_per_test": 3.443111, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 5962.82, "stderr": 0.0, "cost_per_test": 32.108773, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "intellectual_property": { "kimi/kimi-k2.6": { "accuracy": 20.0, "latency": 1400.054, "stderr": 20.0, "cost_per_test": 0.808637, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.7": { "accuracy": 20.0, "latency": 2665.544, "stderr": 20.0, "cost_per_test": 13.072164, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 20.0, "latency": 2805.298, "stderr": 20.0, "cost_per_test": 10.686247, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/ember-1": { "accuracy": 20.0, "latency": 2910.156, "stderr": 20.0, "cost_per_test": 3.950111, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 20.0, "latency": 3064.784, "stderr": 20.0, "cost_per_test": 11.56724, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 65.61, "stderr": 0.0, "cost_per_test": 0.092629, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 91.85, "stderr": 0.0, "cost_per_test": 0.212309, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 126.826, "stderr": 0.0, "cost_per_test": 0.042407, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 129.784, "stderr": 0.0, "cost_per_test": 0.400486, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 136.232, "stderr": 0.0, "cost_per_test": 0.330997, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 179.362, "stderr": 0.0, "cost_per_test": 0.27552, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 265.502, "stderr": 0.0, "cost_per_test": 1.928307, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 279.476, "stderr": 0.0, "cost_per_test": 1.003874, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 329.608, "stderr": 0.0, "cost_per_test": 0.359272, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 362.93, "stderr": 0.0, "cost_per_test": 0.060908, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 376.326, "stderr": 0.0, "cost_per_test": 0.402281, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 380.938, "stderr": 0.0, "cost_per_test": 0.704723, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 395.842, "stderr": 0.0, "cost_per_test": 0.032956, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 410.16, "stderr": 0.0, "cost_per_test": 0.177096, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 432.43, "stderr": 0.0, "cost_per_test": 1.804212, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 474.614, "stderr": 0.0, "cost_per_test": 1.436571, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 520.944, "stderr": 0.0, "cost_per_test": 0.715833, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 525.112, "stderr": 0.0, "cost_per_test": 0.593024, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 546.476, "stderr": 0.0, "cost_per_test": 0.703517, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 547.544, "stderr": 0.0, "cost_per_test": 0.109026, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 0.0, "latency": 561.322, "stderr": 0.0, "cost_per_test": 0.701418, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 564.488, "stderr": 0.0, "cost_per_test": 0.166991, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 610.822, "stderr": 0.0, "cost_per_test": 4.042103, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 628.006, "stderr": 0.0, "cost_per_test": 3.636776, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 634.237, "stderr": 0.0, "cost_per_test": 0.915146, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 636.93, "stderr": 0.0, "cost_per_test": 0.303093, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 781.45, "stderr": 0.0, "cost_per_test": 0.026849, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 0.0, "latency": 808.438, "stderr": 0.0, "cost_per_test": 2.154605, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 821.328, "stderr": 0.0, "cost_per_test": 3.295337, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 836.683, "stderr": 0.0, "cost_per_test": 1.506441, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 840.968, "stderr": 0.0, "cost_per_test": 1.83107, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 903.21, "stderr": 0.0, "cost_per_test": 3.326913, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 924.138, "stderr": 0.0, "cost_per_test": 2.678832, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 939.262, "stderr": 0.0, "cost_per_test": 0.235396, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 958.576, "stderr": 0.0, "cost_per_test": 2.381656, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 960.974, "stderr": 0.0, "cost_per_test": 0.088739, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 1008.958, "stderr": 0.0, "cost_per_test": 4.951983, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1036.242, "stderr": 0.0, "cost_per_test": 0.205832, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1072.964, "stderr": 0.0, "cost_per_test": 1.749857, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 1112.945, "stderr": 0.0, "cost_per_test": 0.39665, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": 1116.818, "stderr": 0.0, "cost_per_test": 2.754711, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 1121.5, "stderr": 0.0, "cost_per_test": 0.055806, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1126.242, "stderr": 0.0, "cost_per_test": 8.439215, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 1126.738, "stderr": 0.0, "cost_per_test": 0.271727, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1246.85, "stderr": 0.0, "cost_per_test": 1.349078, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1248.01, "stderr": 0.0, "cost_per_test": 0.363599, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1316.854, "stderr": 0.0, "cost_per_test": 0.743241, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1397.42, "stderr": 0.0, "cost_per_test": 9.861514, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1399.04, "stderr": 0.0, "cost_per_test": 20.104733, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1406.772, "stderr": 0.0, "cost_per_test": 1.137063, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1413.284, "stderr": 0.0, "cost_per_test": 9.530409, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 1483.872, "stderr": 0.0, "cost_per_test": 1.757206, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1487.716, "stderr": 0.0, "cost_per_test": 0.171603, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 0.0, "latency": 1515.528, "stderr": 0.0, "cost_per_test": 2.100924, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1536.328, "stderr": 0.0, "cost_per_test": 19.843555, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 1545.29, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1874.49, "stderr": 0.0, "cost_per_test": 1.123387, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 1954.208, "stderr": 0.0, "cost_per_test": 3.001191, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 1969.154, "stderr": 0.0, "cost_per_test": 3.617042, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 2027.16, "stderr": 0.0, "cost_per_test": 3.241614, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 2055.65, "stderr": 0.0, "cost_per_test": 0.218964, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.6": { "accuracy": 0.0, "latency": 2073.084, "stderr": 0.0, "cost_per_test": 3.723686, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2120.66, "stderr": 0.0, "cost_per_test": 3.637047, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 2600.29, "stderr": 0.0, "cost_per_test": 0.569544, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 2640.56, "stderr": 0.0, "cost_per_test": 5.422039, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 2660.726, "stderr": 0.0, "cost_per_test": 1.882569, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 3194.55, "stderr": 0.0, "cost_per_test": 23.467391, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3308.745, "stderr": 0.0, "cost_per_test": 20.701364, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3528.822, "stderr": 0.0, "cost_per_test": 17.055475, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 3975.454, "stderr": 0.0, "cost_per_test": 5.12045, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 9064.15, "stderr": 0.0, "cost_per_test": 48.490754, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "international_trade_sanctions": { "meta/muse_spark_1_3": { "accuracy": 20.0, "latency": 1489.08, "stderr": 20.0, "cost_per_test": 1.89638, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 20.0, "latency": 2018.01, "stderr": 20.0, "cost_per_test": 7.007018, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 47.572, "stderr": 0.0, "cost_per_test": 0.051418, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 86.68, "stderr": 0.0, "cost_per_test": 0.180981, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 116.996, "stderr": 0.0, "cost_per_test": 0.296944, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 151.836, "stderr": 0.0, "cost_per_test": 0.040613, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 271.008, "stderr": 0.0, "cost_per_test": 0.333493, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 284.56, "stderr": 0.0, "cost_per_test": 0.166402, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 292.1, "stderr": 0.0, "cost_per_test": 0.228541, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 320.398, "stderr": 0.0, "cost_per_test": 0.035706, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 0.0, "latency": 372.4, "stderr": 0.0, "cost_per_test": 0.355364, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 392.874, "stderr": 0.0, "cost_per_test": 0.428518, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 397.638, "stderr": 0.0, "cost_per_test": 0.026269, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 414.772, "stderr": 0.0, "cost_per_test": 1.880602, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 432.125, "stderr": 0.0, "cost_per_test": 0.641757, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 438.756, "stderr": 0.0, "cost_per_test": 0.359741, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 461.638, "stderr": 0.0, "cost_per_test": 0.642446, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 521.298, "stderr": 0.0, "cost_per_test": 0.564526, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 534.972, "stderr": 0.0, "cost_per_test": 1.312972, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 551.724, "stderr": 0.0, "cost_per_test": 0.590442, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 555.678, "stderr": 0.0, "cost_per_test": 0.163871, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 566.308, "stderr": 0.0, "cost_per_test": 0.257491, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 631.42, "stderr": 0.0, "cost_per_test": 0.093679, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 632.276, "stderr": 0.0, "cost_per_test": 0.653116, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 636.396, "stderr": 0.0, "cost_per_test": 2.359423, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 643.488, "stderr": 0.0, "cost_per_test": 0.03909, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 646.68, "stderr": 0.0, "cost_per_test": 2.683796, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 673.618, "stderr": 0.0, "cost_per_test": 0.52976, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 676.884, "stderr": 0.0, "cost_per_test": 0.129464, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 728.318, "stderr": 0.0, "cost_per_test": 1.85423, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 750.932, "stderr": 0.0, "cost_per_test": 1.851192, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 751.328, "stderr": 0.0, "cost_per_test": 2.96028, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 762.608, "stderr": 0.0, "cost_per_test": 1.028322, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 788.418, "stderr": 0.0, "cost_per_test": 2.340083, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 0.0, "latency": 819.89, "stderr": 0.0, "cost_per_test": 1.928195, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 857.59, "stderr": 0.0, "cost_per_test": 1.591969, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 859.204, "stderr": 0.0, "cost_per_test": 3.921448, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 912.208, "stderr": 0.0, "cost_per_test": 2.317735, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 955.068, "stderr": 0.0, "cost_per_test": 0.06912, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1023.826, "stderr": 0.0, "cost_per_test": 6.235021, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 1052.306, "stderr": 0.0, "cost_per_test": 1.377178, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1055.988, "stderr": 0.0, "cost_per_test": 2.391323, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1095.19, "stderr": 0.0, "cost_per_test": 0.888266, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1108.946, "stderr": 0.0, "cost_per_test": 1.18382, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1147.412, "stderr": 0.0, "cost_per_test": 1.320403, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1239.718, "stderr": 0.0, "cost_per_test": 13.625923, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1246.816, "stderr": 0.0, "cost_per_test": 0.1231, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1357.182, "stderr": 0.0, "cost_per_test": 7.899077, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1375.622, "stderr": 0.0, "cost_per_test": 0.294691, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1470.566, "stderr": 0.0, "cost_per_test": 7.692416, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1563.178, "stderr": 0.0, "cost_per_test": 0.192448, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1627.856, "stderr": 0.0, "cost_per_test": 0.83481, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1639.072, "stderr": 0.0, "cost_per_test": 0.787466, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1640.224, "stderr": 0.0, "cost_per_test": 15.317586, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1780.01, "stderr": 0.0, "cost_per_test": 0.89711, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 1853.572, "stderr": 0.0, "cost_per_test": 2.221756, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 1939.546, "stderr": 0.0, "cost_per_test": 0.023491, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 1963.404, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 1995.074, "stderr": 0.0, "cost_per_test": 1.406803, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 2010.348, "stderr": 0.0, "cost_per_test": 7.464714, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 0.0, "latency": 2085.605, "stderr": 0.0, "cost_per_test": 2.123477, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2095.02, "stderr": 0.0, "cost_per_test": 3.310241, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 2161.244, "stderr": 0.0, "cost_per_test": 0.466223, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 2228.33, "stderr": 0.0, "cost_per_test": 0.192339, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 2245.596, "stderr": 0.0, "cost_per_test": 1.091719, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 2305.496, "stderr": 0.0, "cost_per_test": 3.46041, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2309.216, "stderr": 0.0, "cost_per_test": 2.114472, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.6": { "accuracy": 0.0, "latency": 2372.88, "stderr": 0.0, "cost_per_test": 3.054658, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 2464.414, "stderr": 0.0, "cost_per_test": 4.173567, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2737.024, "stderr": 0.0, "cost_per_test": 10.904646, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 2783.904, "stderr": 0.0, "cost_per_test": 3.380109, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 2972.586, "stderr": 0.0, "cost_per_test": 20.461751, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 3028.722, "stderr": 0.0, "cost_per_test": 28.213056, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3099.466, "stderr": 0.0, "cost_per_test": 20.677557, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3708.434, "stderr": 0.0, "cost_per_test": 16.269947, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } }, "litigation_dispute_resolution": { "grok/grok-4.5": { "accuracy": 20.0, "latency": 666.44, "stderr": 20.0, "cost_per_test": 2.62856, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 20.0, "latency": 852.802, "stderr": 20.0, "cost_per_test": 3.706073, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 20.0, "latency": 1137.476, "stderr": 20.0, "cost_per_test": 0.166693, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 20.0, "latency": 1467.622, "stderr": 20.0, "cost_per_test": 16.209915, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 20.0, "latency": 1622.368, "stderr": 20.0, "cost_per_test": 0.914139, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 2281.146, "stderr": 20.0, "cost_per_test": 0.217599, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 20.0, "latency": 2534.012, "stderr": 20.0, "cost_per_test": 10.15094, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.6": { "accuracy": 20.0, "latency": 2587.706, "stderr": 20.0, "cost_per_test": 4.201399, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/ember-1": { "accuracy": 20.0, "latency": 2616.116, "stderr": 20.0, "cost_per_test": 3.105393, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": 2703.37, "stderr": 20.0, "cost_per_test": 4.259886, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 20.0, "latency": 2828.388, "stderr": 20.0, "cost_per_test": 4.311466, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 20.0, "latency": 3753.636, "stderr": 20.0, "cost_per_test": 2.324679, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 57.228, "stderr": 0.0, "cost_per_test": 0.081949, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 104.598, "stderr": 0.0, "cost_per_test": 0.356731, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 130.01, "stderr": 0.0, "cost_per_test": 0.199512, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 162.686, "stderr": 0.0, "cost_per_test": 0.183489, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 174.518, "stderr": 0.0, "cost_per_test": 0.413946, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 184.048, "stderr": 0.0, "cost_per_test": 0.051964, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 344.866, "stderr": 0.0, "cost_per_test": 0.597517, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 356.668, "stderr": 0.0, "cost_per_test": 0.04251, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 360.998, "stderr": 0.0, "cost_per_test": 0.145792, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 403.072, "stderr": 0.0, "cost_per_test": 2.593296, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 427.864, "stderr": 0.0, "cost_per_test": 0.431199, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 452.15, "stderr": 0.0, "cost_per_test": 0.035311, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 476.208, "stderr": 0.0, "cost_per_test": 0.424558, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 476.506, "stderr": 0.0, "cost_per_test": 0.721268, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 488.464, "stderr": 0.0, "cost_per_test": 0.447406, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 503.798, "stderr": 0.0, "cost_per_test": 0.557082, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 522.006, "stderr": 0.0, "cost_per_test": 2.378166, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 560.644, "stderr": 0.0, "cost_per_test": 3.560829, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 580.87, "stderr": 0.0, "cost_per_test": 0.103181, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 582.58, "stderr": 0.0, "cost_per_test": 0.18267, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 596.268, "stderr": 0.0, "cost_per_test": 6.932258, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 636.384, "stderr": 0.0, "cost_per_test": 0.488931, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 642.88, "stderr": 0.0, "cost_per_test": 0.187536, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 684.738, "stderr": 0.0, "cost_per_test": 0.024251, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 0.0, "latency": 748.106, "stderr": 0.0, "cost_per_test": 0.879933, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 749.656, "stderr": 0.0, "cost_per_test": 2.138974, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 772.505, "stderr": 0.0, "cost_per_test": 1.156532, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 821.022, "stderr": 0.0, "cost_per_test": 0.997521, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 824.628, "stderr": 0.0, "cost_per_test": 4.460204, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 854.906, "stderr": 0.0, "cost_per_test": 1.378443, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 0.0, "latency": 875.784, "stderr": 0.0, "cost_per_test": 2.376132, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 908.618, "stderr": 0.0, "cost_per_test": 7.56413, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 941.828, "stderr": 0.0, "cost_per_test": 9.61742, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 946.888, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 1024.006, "stderr": 0.0, "cost_per_test": 3.252605, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1029.628, "stderr": 0.0, "cost_per_test": 2.465192, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": 1032.806, "stderr": 0.0, "cost_per_test": 3.090873, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 1043.246, "stderr": 0.0, "cost_per_test": 0.089255, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1137.594, "stderr": 0.0, "cost_per_test": 1.209207, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 0.0, "latency": 1195.492, "stderr": 0.0, "cost_per_test": 2.134092, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1227.488, "stderr": 0.0, "cost_per_test": 0.345901, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1335.276, "stderr": 0.0, "cost_per_test": 9.535296, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1390.64, "stderr": 0.0, "cost_per_test": 0.302625, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1402.92, "stderr": 0.0, "cost_per_test": 1.540985, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 1415.958, "stderr": 0.0, "cost_per_test": 0.055896, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1443.394, "stderr": 0.0, "cost_per_test": 2.117782, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1464.802, "stderr": 0.0, "cost_per_test": 10.4087, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 1498.17, "stderr": 0.0, "cost_per_test": 1.41634, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 1524.202, "stderr": 0.0, "cost_per_test": 0.261866, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1542.606, "stderr": 0.0, "cost_per_test": 33.042054, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 1585.042, "stderr": 0.0, "cost_per_test": 0.137375, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1624.24, "stderr": 0.0, "cost_per_test": 0.908831, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1720.822, "stderr": 0.0, "cost_per_test": 0.94901, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 1914.198, "stderr": 0.0, "cost_per_test": 8.481032, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2109.852, "stderr": 0.0, "cost_per_test": 3.610666, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 2372.292, "stderr": 0.0, "cost_per_test": 4.594796, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2495.782, "stderr": 0.0, "cost_per_test": 4.394844, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 2867.042, "stderr": 0.0, "cost_per_test": 19.378797, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 2932.168, "stderr": 0.0, "cost_per_test": 10.28651, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 2956.134, "stderr": 0.0, "cost_per_test": 13.912957, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3186.104, "stderr": 0.0, "cost_per_test": 21.148726, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 3576.986, "stderr": 0.0, "cost_per_test": 0.539041, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 3775.334, "stderr": 0.0, "cost_per_test": 5.593312, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 5938.524, "stderr": 0.0, "cost_per_test": 41.44865, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "real_estate": { "grok/grok-4.5": { "accuracy": 20.0, "latency": 625.732, "stderr": 20.0, "cost_per_test": 1.767673, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 20.0, "latency": 853.466, "stderr": 20.0, "cost_per_test": 2.754558, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 20.0, "latency": 934.998, "stderr": 20.0, "cost_per_test": 0.95827, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.2": { "accuracy": 20.0, "latency": 1153.204, "stderr": 20.0, "cost_per_test": 1.899249, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 20.0, "latency": 1388.648, "stderr": 20.0, "cost_per_test": 5.113504, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 20.0, "latency": 1568.462, "stderr": 20.0, "cost_per_test": 10.93977, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 20.0, "latency": 1681.804, "stderr": 20.0, "cost_per_test": 2.7908, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "tencent/hy4-preview": { "accuracy": 20.0, "latency": 1817.306, "stderr": 20.0, "cost_per_test": 1.123838, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": 1969.466, "stderr": 20.0, "cost_per_test": 3.571062, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 20.0, "latency": 2428.202, "stderr": 20.0, "cost_per_test": 4.477501, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 82.088, "stderr": 0.0, "cost_per_test": 0.20807, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 188.604, "stderr": 0.0, "cost_per_test": 0.060815, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 217.598, "stderr": 0.0, "cost_per_test": 0.312805, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 262.84, "stderr": 0.0, "cost_per_test": 0.886902, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 285.042, "stderr": 0.0, "cost_per_test": 0.33031, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 286.224, "stderr": 0.0, "cost_per_test": 0.203965, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 367.182, "stderr": 0.0, "cost_per_test": 0.050466, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 381.558, "stderr": 0.0, "cost_per_test": 2.814369, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 387.812, "stderr": 0.0, "cost_per_test": 0.437245, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 391.026, "stderr": 0.0, "cost_per_test": 0.461787, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 397.494, "stderr": 0.0, "cost_per_test": 0.444015, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 406.892, "stderr": 0.0, "cost_per_test": 0.50823, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 414.664, "stderr": 0.0, "cost_per_test": 0.040377, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 415.102, "stderr": 0.0, "cost_per_test": 0.804107, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 415.564, "stderr": 0.0, "cost_per_test": 0.175367, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 417.46, "stderr": 0.0, "cost_per_test": 0.193926, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 432.99, "stderr": 0.0, "cost_per_test": 0.200928, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 501.908, "stderr": 0.0, "cost_per_test": 2.425857, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 516.24, "stderr": 0.0, "cost_per_test": 0.54794, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 546.054, "stderr": 0.0, "cost_per_test": 3.068252, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 589.834, "stderr": 0.0, "cost_per_test": 0.127716, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 699.528, "stderr": 0.0, "cost_per_test": 0.486947, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 751.535, "stderr": 0.0, "cost_per_test": 0.96877, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 799.052, "stderr": 0.0, "cost_per_test": 5.565429, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 802.4, "stderr": 0.0, "cost_per_test": 0.362861, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 841.228, "stderr": 0.0, "cost_per_test": 3.880679, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 939.142, "stderr": 0.0, "cost_per_test": 0.066263, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 1000.01, "stderr": 0.0, "cost_per_test": 4.665221, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 1010.134, "stderr": 0.0, "cost_per_test": 2.121529, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 1012.23, "stderr": 0.0, "cost_per_test": 4.677799, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 1046.09, "stderr": 0.0, "cost_per_test": 3.508749, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 1090.508, "stderr": 0.0, "cost_per_test": 4.771601, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1101.184, "stderr": 0.0, "cost_per_test": 0.893435, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1116.426, "stderr": 0.0, "cost_per_test": 2.958384, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1155.776, "stderr": 0.0, "cost_per_test": 0.51747, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 1190.406, "stderr": 0.0, "cost_per_test": 0.101049, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1362.7, "stderr": 0.0, "cost_per_test": 1.394793, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 1414.578, "stderr": 0.0, "cost_per_test": 0.122737, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1430.388, "stderr": 0.0, "cost_per_test": 0.366047, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1485.644, "stderr": 0.0, "cost_per_test": 10.731391, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1535.996, "stderr": 0.0, "cost_per_test": 0.19941, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1616.44, "stderr": 0.0, "cost_per_test": 25.792468, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1656.608, "stderr": 0.0, "cost_per_test": 1.096893, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1678.198, "stderr": 0.0, "cost_per_test": 21.067576, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 1700.696, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1732.598, "stderr": 0.0, "cost_per_test": 12.830219, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 1886.13, "stderr": 0.0, "cost_per_test": 1.775065, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1936.218, "stderr": 0.0, "cost_per_test": 1.163256, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.6": { "accuracy": 0.0, "latency": 2073.896, "stderr": 0.0, "cost_per_test": 3.122275, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2101.336, "stderr": 0.0, "cost_per_test": 4.008978, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 2122.326, "stderr": 0.0, "cost_per_test": 0.266735, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 2252.274, "stderr": 0.0, "cost_per_test": 0.414459, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 2259.924, "stderr": 0.0, "cost_per_test": 4.5574, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2298.07, "stderr": 0.0, "cost_per_test": 9.655661, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2412.168, "stderr": 0.0, "cost_per_test": 5.007137, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 2771.174, "stderr": 0.0, "cost_per_test": 0.038282, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 2920.534, "stderr": 0.0, "cost_per_test": 10.465474, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 2942.99, "stderr": 0.0, "cost_per_test": 21.779478, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 3242.052, "stderr": 0.0, "cost_per_test": 4.710728, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3371.66, "stderr": 0.0, "cost_per_test": 16.877102, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 3422.408, "stderr": 0.0, "cost_per_test": 2.96409, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 3512.384, "stderr": 0.0, "cost_per_test": 16.654201, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 3620.802, "stderr": 0.0, "cost_per_test": 4.871351, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 3661.252, "stderr": 0.0, "cost_per_test": 4.367429, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 3945.164, "stderr": 0.0, "cost_per_test": 28.957365, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 8514.508, "stderr": 0.0, "cost_per_test": 54.321067, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "structured_finance_securitization": { "kimi/kimi-k3": { "accuracy": 20.0, "latency": 519.34, "stderr": 20.0, "cost_per_test": 3.636689, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 20.0, "latency": 532.566, "stderr": 20.0, "cost_per_test": 1.638501, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 20.0, "latency": 565.392, "stderr": 20.0, "cost_per_test": 0.630492, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 20.0, "latency": 772.732, "stderr": 20.0, "cost_per_test": 2.165683, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 20.0, "latency": 783.044, "stderr": 20.0, "cost_per_test": 0.055487, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 20.0, "latency": 1052.35, "stderr": 20.0, "cost_per_test": 3.686565, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 20.0, "latency": 1102.334, "stderr": 20.0, "cost_per_test": 1.625328, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.2": { "accuracy": 20.0, "latency": 1144.854, "stderr": 20.0, "cost_per_test": 1.925613, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 20.0, "latency": 1487.61, "stderr": 20.0, "cost_per_test": 11.288588, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 20.0, "latency": 1649.426, "stderr": 20.0, "cost_per_test": 0.092304, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 20.0, "latency": 1695.05, "stderr": 20.0, "cost_per_test": 0.84184, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 20.0, "latency": 1767.882, "stderr": 20.0, "cost_per_test": 3.302349, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 20.0, "latency": 2122.848, "stderr": 20.0, "cost_per_test": 3.845931, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 2279.76, "stderr": 20.0, "cost_per_test": 0.212322, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 20.0, "latency": 2423.536, "stderr": 20.0, "cost_per_test": 3.983856, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/ember-1": { "accuracy": 20.0, "latency": 2815.95, "stderr": 20.0, "cost_per_test": 3.196836, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 20.0, "latency": 3180.94, "stderr": 20.0, "cost_per_test": 0.655247, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 20.0, "latency": 3725.94, "stderr": 20.0, "cost_per_test": 27.562306, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 55.102, "stderr": 0.0, "cost_per_test": 0.066982, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 79.682, "stderr": 0.0, "cost_per_test": 0.225971, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 145.977, "stderr": 0.0, "cost_per_test": 0.15069, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 155.73, "stderr": 0.0, "cost_per_test": 0.04442, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 185.046, "stderr": 0.0, "cost_per_test": 0.388122, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 206.166, "stderr": 0.0, "cost_per_test": 0.29865, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 222.714, "stderr": 0.0, "cost_per_test": 0.253469, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 266.99, "stderr": 0.0, "cost_per_test": 0.326243, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 273.01, "stderr": 0.0, "cost_per_test": 0.312857, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 320.916, "stderr": 0.0, "cost_per_test": 0.381923, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 326.596, "stderr": 0.0, "cost_per_test": 0.038287, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 353.186, "stderr": 0.0, "cost_per_test": 0.195145, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 365.932, "stderr": 0.0, "cost_per_test": 0.039132, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 385.458, "stderr": 0.0, "cost_per_test": 0.396449, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 393.294, "stderr": 0.0, "cost_per_test": 0.786036, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 484.552, "stderr": 0.0, "cost_per_test": 0.11464, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 561.12, "stderr": 0.0, "cost_per_test": 3.015746, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 564.454, "stderr": 0.0, "cost_per_test": 0.186047, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 578.878, "stderr": 0.0, "cost_per_test": 1.902953, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 629.124, "stderr": 0.0, "cost_per_test": 1.246746, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 651.854, "stderr": 0.0, "cost_per_test": 0.424212, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 663.878, "stderr": 0.0, "cost_per_test": 2.811066, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 663.972, "stderr": 0.0, "cost_per_test": 0.917778, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 678.652, "stderr": 0.0, "cost_per_test": 2.18141, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 752.314, "stderr": 0.0, "cost_per_test": 2.587486, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 779.112, "stderr": 0.0, "cost_per_test": 3.145583, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 794.386, "stderr": 0.0, "cost_per_test": 7.609913, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 805.298, "stderr": 0.0, "cost_per_test": 1.74196, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 833.975, "stderr": 0.0, "cost_per_test": 1.118034, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 842.814, "stderr": 0.0, "cost_per_test": 2.506224, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 870.666, "stderr": 0.0, "cost_per_test": 1.692267, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 925.634, "stderr": 0.0, "cost_per_test": 1.250112, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1117.962, "stderr": 0.0, "cost_per_test": 3.023182, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1148.782, "stderr": 0.0, "cost_per_test": 0.254479, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1252.694, "stderr": 0.0, "cost_per_test": 14.536468, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1313.738, "stderr": 0.0, "cost_per_test": 9.320472, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1320.054, "stderr": 0.0, "cost_per_test": 0.163846, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 1378.032, "stderr": 0.0, "cost_per_test": 5.98365, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1394.672, "stderr": 0.0, "cost_per_test": 0.382612, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1461.506, "stderr": 0.0, "cost_per_test": 23.080257, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 1494.943, "stderr": 0.0, "cost_per_test": 0.312124, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1580.746, "stderr": 0.0, "cost_per_test": 17.134562, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1699.996, "stderr": 0.0, "cost_per_test": 0.903082, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 1812.578, "stderr": 0.0, "cost_per_test": 1.250902, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1816.484, "stderr": 0.0, "cost_per_test": 0.983278, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 1934.564, "stderr": 0.0, "cost_per_test": 0.169113, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 2100.302, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2118.808, "stderr": 0.0, "cost_per_test": 3.463381, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2318.596, "stderr": 0.0, "cost_per_test": 9.939067, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 2366.126, "stderr": 0.0, "cost_per_test": 0.022576, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 2425.614, "stderr": 0.0, "cost_per_test": 4.677121, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 2674.492, "stderr": 0.0, "cost_per_test": 2.178978, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 2851.368, "stderr": 0.0, "cost_per_test": 4.509767, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 2994.183, "stderr": 0.0, "cost_per_test": 13.720065, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 3112.574, "stderr": 0.0, "cost_per_test": 4.517092, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 3445.546, "stderr": 0.0, "cost_per_test": 13.409945, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3641.718, "stderr": 0.0, "cost_per_test": 19.757045, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 6243.388, "stderr": 0.0, "cost_per_test": 41.180587, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "tax": { "meta/muse_spark_1_2": { "accuracy": 20.0, "latency": 1835.168, "stderr": 20.0, "cost_per_test": 2.558771, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 20.0, "latency": 3286.746, "stderr": 20.0, "cost_per_test": 6.118001, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 61.968, "stderr": 0.0, "cost_per_test": 0.08909, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 75.256, "stderr": 0.0, "cost_per_test": 0.179616, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 141.542, "stderr": 0.0, "cost_per_test": 0.039851, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 179.408, "stderr": 0.0, "cost_per_test": 0.380436, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 231.336, "stderr": 0.0, "cost_per_test": 0.251258, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 300.574, "stderr": 0.0, "cost_per_test": 0.312585, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 313.794, "stderr": 0.0, "cost_per_test": 0.24687, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 381.644, "stderr": 0.0, "cost_per_test": 0.767561, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 390.0, "stderr": 0.0, "cost_per_test": 0.044615, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 391.522, "stderr": 0.0, "cost_per_test": 2.850433, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 394.79, "stderr": 0.0, "cost_per_test": 0.628204, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 404.446, "stderr": 0.0, "cost_per_test": 0.177002, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 432.008, "stderr": 0.0, "cost_per_test": 0.533038, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 456.764, "stderr": 0.0, "cost_per_test": 0.530065, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 497.872, "stderr": 0.0, "cost_per_test": 0.041821, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 502.124, "stderr": 0.0, "cost_per_test": 0.198778, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 530.32, "stderr": 0.0, "cost_per_test": 2.60193, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 531.476, "stderr": 0.0, "cost_per_test": 1.829973, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 547.97, "stderr": 0.0, "cost_per_test": 2.239645, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 574.202, "stderr": 0.0, "cost_per_test": 0.106054, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 592.95, "stderr": 0.0, "cost_per_test": 0.670951, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 662.032, "stderr": 0.0, "cost_per_test": 0.805207, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 671.384, "stderr": 0.0, "cost_per_test": 4.83132, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 0.0, "latency": 698.208, "stderr": 0.0, "cost_per_test": 0.714588, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 709.58, "stderr": 0.0, "cost_per_test": 4.513818, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 720.268, "stderr": 0.0, "cost_per_test": 0.923588, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 778.23, "stderr": 0.0, "cost_per_test": 0.297344, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 794.508, "stderr": 0.0, "cost_per_test": 0.596364, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 837.746, "stderr": 0.0, "cost_per_test": 0.915144, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 860.462, "stderr": 0.0, "cost_per_test": 4.446485, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 873.126, "stderr": 0.0, "cost_per_test": 2.661317, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": 892.478, "stderr": 0.0, "cost_per_test": 2.323616, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 908.22, "stderr": 0.0, "cost_per_test": 0.096872, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 913.98, "stderr": 0.0, "cost_per_test": 3.459839, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 961.292, "stderr": 0.0, "cost_per_test": 0.065423, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 964.33, "stderr": 0.0, "cost_per_test": 0.358568, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 0.0, "latency": 1040.15, "stderr": 0.0, "cost_per_test": 2.635161, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 1063.3, "stderr": 0.0, "cost_per_test": 1.480887, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1159.824, "stderr": 0.0, "cost_per_test": 10.025086, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 1207.288, "stderr": 0.0, "cost_per_test": 5.647333, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1344.49, "stderr": 0.0, "cost_per_test": 0.247354, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 1392.034, "stderr": 0.0, "cost_per_test": 3.796478, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1412.566, "stderr": 0.0, "cost_per_test": 4.269538, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1470.598, "stderr": 0.0, "cost_per_test": 1.553621, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1497.818, "stderr": 0.0, "cost_per_test": 0.181593, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1637.674, "stderr": 0.0, "cost_per_test": 2.610141, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1655.896, "stderr": 0.0, "cost_per_test": 47.22613, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1692.904, "stderr": 0.0, "cost_per_test": 9.85827, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1716.128, "stderr": 0.0, "cost_per_test": 0.890846, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1733.256, "stderr": 0.0, "cost_per_test": 10.87057, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1763.37, "stderr": 0.0, "cost_per_test": 17.075748, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1861.222, "stderr": 0.0, "cost_per_test": 1.092562, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 1875.724, "stderr": 0.0, "cost_per_test": 0.167663, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 1979.525, "stderr": 0.0, "cost_per_test": 2.288379, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 2045.018, "stderr": 0.0, "cost_per_test": 0.403124, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 2140.868, "stderr": 0.0, "cost_per_test": 10.699038, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 2473.116, "stderr": 0.0, "cost_per_test": 1.620848, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 2489.228, "stderr": 0.0, "cost_per_test": 1.290764, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 2605.465, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2624.66, "stderr": 0.0, "cost_per_test": 8.873133, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 2787.138, "stderr": 0.0, "cost_per_test": 3.914791, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 2790.458, "stderr": 0.0, "cost_per_test": 10.875562, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 0.0, "latency": 2936.336, "stderr": 0.0, "cost_per_test": 4.720799, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 3155.744, "stderr": 0.0, "cost_per_test": 2.991236, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3256.682, "stderr": 0.0, "cost_per_test": 17.274333, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 3328.596, "stderr": 0.0, "cost_per_test": 0.714506, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 3432.134, "stderr": 0.0, "cost_per_test": 4.824852, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3703.608, "stderr": 0.0, "cost_per_test": 16.089025, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 3725.868, "stderr": 0.0, "cost_per_test": 6.356447, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 3939.954, "stderr": 0.0, "cost_per_test": 0.060202, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 4204.678, "stderr": 0.0, "cost_per_test": 26.870915, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 4209.854, "stderr": 0.0, "cost_per_test": 8.046596, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 5267.096, "stderr": 0.0, "cost_per_test": 6.478837, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 8351.112, "stderr": 0.0, "cost_per_test": 47.644121, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "trusts_estates_private_client": { "google/gemini-4-argon": { "accuracy": 40.0, "latency": 3234.098, "stderr": 24.495, "cost_per_test": 16.183565, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 20.0, "latency": 436.982, "stderr": 20.0, "cost_per_test": 0.15293, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 20.0, "latency": 530.524, "stderr": 20.0, "cost_per_test": 2.192393, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 20.0, "latency": 613.212, "stderr": 20.0, "cost_per_test": 1.84661, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 20.0, "latency": 613.88, "stderr": 20.0, "cost_per_test": 0.719941, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 20.0, "latency": 951.492, "stderr": 20.0, "cost_per_test": 0.088474, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 20.0, "latency": 979.01, "stderr": 20.0, "cost_per_test": 1.596615, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 20.0, "latency": 985.068, "stderr": 20.0, "cost_per_test": 2.033707, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 20.0, "latency": 1037.778, "stderr": 20.0, "cost_per_test": 1.784859, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 20.0, "latency": 1314.758, "stderr": 20.0, "cost_per_test": 1.805781, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-astra": { "accuracy": 20.0, "latency": 1477.192, "stderr": 20.0, "cost_per_test": 23.645844, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 20.0, "latency": 1662.664, "stderr": 20.0, "cost_per_test": 18.186942, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 20.0, "latency": 2718.166, "stderr": 20.0, "cost_per_test": 4.184536, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 20.0, "latency": 3244.644, "stderr": 20.0, "cost_per_test": 15.106108, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 45.062, "stderr": 0.0, "cost_per_test": 0.043684, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 81.35, "stderr": 0.0, "cost_per_test": 0.209388, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 104.812, "stderr": 0.0, "cost_per_test": 0.232718, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 170.998, "stderr": 0.0, "cost_per_test": 0.038531, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 187.458, "stderr": 0.0, "cost_per_test": 0.223928, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 276.672, "stderr": 0.0, "cost_per_test": 0.974394, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 280.702, "stderr": 0.0, "cost_per_test": 0.035772, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 323.742, "stderr": 0.0, "cost_per_test": 0.352488, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 351.656, "stderr": 0.0, "cost_per_test": 2.37985, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 371.812, "stderr": 0.0, "cost_per_test": 0.188846, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 400.142, "stderr": 0.0, "cost_per_test": 0.400358, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 423.01, "stderr": 0.0, "cost_per_test": 0.467668, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 432.742, "stderr": 0.0, "cost_per_test": 0.175098, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 462.122, "stderr": 0.0, "cost_per_test": 0.765564, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 471.38, "stderr": 0.0, "cost_per_test": 0.498053, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 484.966, "stderr": 0.0, "cost_per_test": 0.526516, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 485.3, "stderr": 0.0, "cost_per_test": 0.041019, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 505.312, "stderr": 0.0, "cost_per_test": 0.25343, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 541.676, "stderr": 0.0, "cost_per_test": 0.115297, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 684.416, "stderr": 0.0, "cost_per_test": 4.928295, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 753.492, "stderr": 0.0, "cost_per_test": 0.887587, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 774.15, "stderr": 0.0, "cost_per_test": 2.72662, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 810.77, "stderr": 0.0, "cost_per_test": 4.099425, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 840.962, "stderr": 0.0, "cost_per_test": 2.21818, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 883.526, "stderr": 0.0, "cost_per_test": 3.626126, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 917.72, "stderr": 0.0, "cost_per_test": 0.198018, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 952.646, "stderr": 0.0, "cost_per_test": 3.743606, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 982.602, "stderr": 0.0, "cost_per_test": 0.226872, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 995.132, "stderr": 0.0, "cost_per_test": 0.039247, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 1068.532, "stderr": 0.0, "cost_per_test": 5.065086, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1109.494, "stderr": 0.0, "cost_per_test": 2.162434, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 1141.15, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1146.278, "stderr": 0.0, "cost_per_test": 0.6436, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 1151.496, "stderr": 0.0, "cost_per_test": 2.752136, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 1181.327, "stderr": 0.0, "cost_per_test": 0.059982, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1249.462, "stderr": 0.0, "cost_per_test": 0.23816, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1270.302, "stderr": 0.0, "cost_per_test": 0.74451, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1290.308, "stderr": 0.0, "cost_per_test": 2.740693, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1292.3, "stderr": 0.0, "cost_per_test": 0.163816, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1340.898, "stderr": 0.0, "cost_per_test": 9.820962, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1349.698, "stderr": 0.0, "cost_per_test": 11.396437, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1368.64, "stderr": 0.0, "cost_per_test": 8.582153, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 1441.398, "stderr": 0.0, "cost_per_test": 0.305605, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1459.198, "stderr": 0.0, "cost_per_test": 1.392685, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1659.25, "stderr": 0.0, "cost_per_test": 0.915369, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1796.99, "stderr": 0.0, "cost_per_test": 2.467191, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 1942.364, "stderr": 0.0, "cost_per_test": 2.935175, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 1990.218, "stderr": 0.0, "cost_per_test": 3.065155, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 2018.478, "stderr": 0.0, "cost_per_test": 3.714805, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 2288.172, "stderr": 0.0, "cost_per_test": 0.177119, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 2344.854, "stderr": 0.0, "cost_per_test": 1.287155, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 0.0, "latency": 2647.598, "stderr": 0.0, "cost_per_test": 4.653439, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2788.88, "stderr": 0.0, "cost_per_test": 12.4848, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 2937.012, "stderr": 0.0, "cost_per_test": 0.541093, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 3045.714, "stderr": 0.0, "cost_per_test": 5.517077, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 3106.68, "stderr": 0.0, "cost_per_test": 24.97725, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 3221.54, "stderr": 0.0, "cost_per_test": 9.687818, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3375.65, "stderr": 0.0, "cost_per_test": 21.781833, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 3477.792, "stderr": 0.0, "cost_per_test": 4.018375, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 3696.17, "stderr": 0.0, "cost_per_test": 2.595506, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 3814.938, "stderr": 0.0, "cost_per_test": 6.292927, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 8870.548, "stderr": 0.0, "cost_per_test": 47.297443, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "white_collar_defense_investigations": { "google/gemini-3.1-flash-lite-preview": { "accuracy": 0.0, "latency": 47.606, "stderr": 0.0, "cost_per_test": 0.072372, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 0.0, "latency": 119.378, "stderr": 0.0, "cost_per_test": 0.176704, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 0.0, "latency": 136.074, "stderr": 0.0, "cost_per_test": 0.243093, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 0.0, "latency": 140.312, "stderr": 0.0, "cost_per_test": 0.311913, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 0.0, "latency": 174.8, "stderr": 0.0, "cost_per_test": 0.002798, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 0.0, "latency": 192.164, "stderr": 0.0, "cost_per_test": 0.043819, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 0.0, "latency": 275.234, "stderr": 0.0, "cost_per_test": 0.665432, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 0.0, "latency": 282.53, "stderr": 0.0, "cost_per_test": 0.260621, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 0.0, "latency": 286.138, "stderr": 0.0, "cost_per_test": 0.345186, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 0.0, "latency": 301.998, "stderr": 0.0, "cost_per_test": 0.135545, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 0.0, "latency": 307.222, "stderr": 0.0, "cost_per_test": 0.379403, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 0.0, "latency": 324.058, "stderr": 0.0, "cost_per_test": 0.148109, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 0.0, "latency": 339.398, "stderr": 0.0, "cost_per_test": 0.03941, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 0.0, "latency": 380.548, "stderr": 0.0, "cost_per_test": 1.771686, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 0.0, "latency": 389.944, "stderr": 0.0, "cost_per_test": 0.597321, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 0.0, "latency": 439.742, "stderr": 0.0, "cost_per_test": 0.028794, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": 440.762, "stderr": 0.0, "cost_per_test": 0.136331, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 0.0, "latency": 445.292, "stderr": 0.0, "cost_per_test": 0.588553, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 0.0, "latency": 532.962, "stderr": 0.0, "cost_per_test": 0.739764, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.5": { "accuracy": 0.0, "latency": 571.98, "stderr": 0.0, "cost_per_test": 3.615019, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 0.0, "latency": 590.94, "stderr": 0.0, "cost_per_test": 0.522794, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 0.0, "latency": 603.842, "stderr": 0.0, "cost_per_test": 0.099168, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 0.0, "latency": 605.6, "stderr": 0.0, "cost_per_test": 0.303629, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 0.0, "latency": 623.543, "stderr": 0.0, "cost_per_test": 0.976564, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 630.426, "stderr": 0.0, "cost_per_test": 4.016727, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 0.0, "latency": 654.63, "stderr": 0.0, "cost_per_test": 0.053956, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 0.0, "latency": 665.522, "stderr": 0.0, "cost_per_test": 1.635544, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 0.0, "latency": 683.694, "stderr": 0.0, "cost_per_test": 2.379245, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 0.0, "latency": 708.492, "stderr": 0.0, "cost_per_test": 2.297215, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 0.0, "latency": 713.156, "stderr": 0.0, "cost_per_test": 1.399547, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 0.0, "latency": 732.282, "stderr": 0.0, "cost_per_test": 2.683246, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 758.704, "stderr": 0.0, "cost_per_test": 0.07138, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 0.0, "latency": 772.198, "stderr": 0.0, "cost_per_test": 0.146247, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 0.0, "latency": 805.922, "stderr": 0.0, "cost_per_test": 3.049149, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 0.0, "latency": 806.106, "stderr": 0.0, "cost_per_test": 4.60707, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 0.0, "latency": 896.148, "stderr": 0.0, "cost_per_test": 3.236421, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 0.0, "latency": 906.196, "stderr": 0.0, "cost_per_test": 2.137307, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1020.596, "stderr": 0.0, "cost_per_test": 0.16563, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1069.216, "stderr": 0.0, "cost_per_test": 0.148855, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 1084.252, "stderr": 0.0, "cost_per_test": 1.451903, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 0.0, "latency": 1094.13, "stderr": 0.0, "cost_per_test": 2.939674, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 0.0, "latency": 1094.404, "stderr": 0.0, "cost_per_test": 2.560117, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 0.0, "latency": 1106.794, "stderr": 0.0, "cost_per_test": 1.117058, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5.2": { "accuracy": 0.0, "latency": 1118.582, "stderr": 0.0, "cost_per_test": 1.565734, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 0.0, "latency": 1127.786, "stderr": 0.0, "cost_per_test": 2.783077, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 0.0, "latency": 1149.156, "stderr": 0.0, "cost_per_test": 0.592721, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 0.0, "latency": 1199.134, "stderr": 0.0, "cost_per_test": 10.413192, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 0.0, "latency": 1200.23, "stderr": 0.0, "cost_per_test": 9.056848, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 0.0, "latency": 1278.39, "stderr": 0.0, "cost_per_test": 1.579724, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 0.0, "latency": 1340.046, "stderr": 0.0, "cost_per_test": 10.000136, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 1340.72, "stderr": 0.0, "cost_per_test": 0.15481, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "stepfun/step-5-preview": { "accuracy": 0.0, "latency": 1482.306, "stderr": 0.0, "cost_per_test": 0.835096, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1489.37, "stderr": 0.0, "cost_per_test": 0.473577, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1512.602, "stderr": 0.0, "cost_per_test": 23.924481, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 0.0, "latency": 1555.02, "stderr": 0.0, "cost_per_test": 16.626886, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 1588.488, "stderr": 0.0, "cost_per_test": 2.297079, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 0.0, "latency": 1720.744, "stderr": 0.0, "cost_per_test": 2.154821, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1775.898, "stderr": 0.0, "cost_per_test": 0.869928, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 1775.954, "stderr": 0.0, "cost_per_test": 3.154232, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 0.0, "latency": 1835.934, "stderr": 0.0, "cost_per_test": 5.191854, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 0.0, "latency": 1864.574, "stderr": 0.0, "cost_per_test": 1.435625, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 2041.394, "stderr": 0.0, "cost_per_test": 3.121405, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 0.0, "latency": 2104.122, "stderr": 0.0, "cost_per_test": 0.366044, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 2107.032, "stderr": 0.0, "cost_per_test": 0.325751, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 2215.2, "stderr": 0.0, "cost_per_test": 10.940583, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 2273.904, "stderr": 0.0, "cost_per_test": 3.663112, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2284.886, "stderr": 0.0, "cost_per_test": 3.632901, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 0.0, "latency": 2326.0, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2354.074, "stderr": 0.0, "cost_per_test": 10.468913, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 0.0, "latency": 2613.808, "stderr": 0.0, "cost_per_test": 16.147201, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 0.0, "latency": 2812.684, "stderr": 0.0, "cost_per_test": 6.176326, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 0.0, "latency": 2829.624, "stderr": 0.0, "cost_per_test": 2.643192, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.6": { "accuracy": 0.0, "latency": 3257.702, "stderr": 0.0, "cost_per_test": 3.343428, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3362.74, "stderr": 0.0, "cost_per_test": 24.223778, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3657.376, "stderr": 0.0, "cost_per_test": 17.206836, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 0.0, "latency": 8879.844, "stderr": 0.0, "cost_per_test": 42.363937, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "criteria_pass_rate__antitrust_competition": { "meta/muse_spark_1_3": { "accuracy": 96.377, "latency": 923.89, "stderr": 1.127, "cost_per_test": 2.724467, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 95.29, "latency": 1432.75, "stderr": 1.278, "cost_per_test": 1.84335, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 94.565, "latency": 857.572, "stderr": 1.367, "cost_per_test": 2.562941, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 94.203, "latency": 708.48, "stderr": 1.409, "cost_per_test": 0.782306, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 94.203, "latency": 3686.716, "stderr": 1.409, "cost_per_test": 15.535189, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 93.116, "latency": 880.526, "stderr": 1.527, "cost_per_test": 4.295226, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 93.116, "latency": 1808.052, "stderr": 1.527, "cost_per_test": 3.522356, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 92.391, "latency": 2522.454, "stderr": 1.599, "cost_per_test": 3.772087, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 92.035, "latency": 1501.212, "stderr": 1.805, "cost_per_test": 16.832828, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 91.667, "latency": 1972.514, "stderr": 1.667, "cost_per_test": 0.192669, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 91.667, "latency": 2336.496, "stderr": 1.667, "cost_per_test": 10.647354, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 91.304, "latency": 723.782, "stderr": 1.699, "cost_per_test": 0.052986, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 91.304, "latency": 933.252, "stderr": 1.699, "cost_per_test": 0.077418, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 90.942, "latency": 2680.208, "stderr": 1.731, "cost_per_test": 1.955704, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 90.217, "latency": 1311.106, "stderr": 1.791, "cost_per_test": 9.124408, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 89.855, "latency": 1519.004, "stderr": 1.821, "cost_per_test": 0.789632, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 89.855, "latency": 2090.638, "stderr": 1.821, "cost_per_test": 3.909019, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 89.855, "latency": 6352.144, "stderr": 1.821, "cost_per_test": 39.631516, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 89.493, "latency": 517.982, "stderr": 1.849, "cost_per_test": 2.535802, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 89.493, "latency": 2421.988, "stderr": 1.849, "cost_per_test": 3.497776, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 88.768, "latency": 545.892, "stderr": 1.904, "cost_per_test": 2.679324, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 88.768, "latency": 1474.888, "stderr": 1.904, "cost_per_test": 24.979276, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 88.768, "latency": 2025.322, "stderr": 1.904, "cost_per_test": 6.844893, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 88.043, "latency": 2610.216, "stderr": 1.957, "cost_per_test": 2.722361, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 87.681, "latency": 746.252, "stderr": 1.982, "cost_per_test": 0.170347, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 87.681, "latency": 1839.778, "stderr": 1.982, "cost_per_test": 3.871498, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 87.319, "latency": 651.426, "stderr": 2.007, "cost_per_test": 2.727866, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 87.319, "latency": 994.292, "stderr": 2.007, "cost_per_test": 2.673062, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 87.319, "latency": 1101.62, "stderr": 2.007, "cost_per_test": 2.027261, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 87.319, "latency": 1337.142, "stderr": 2.007, "cost_per_test": 9.296892, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 87.319, "latency": 2632.506, "stderr": 2.007, "cost_per_test": 19.151916, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 86.594, "latency": 2674.686, "stderr": 2.055, "cost_per_test": 0.45672, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 85.87, "latency": 1062.228, "stderr": 2.101, "cost_per_test": 0.153299, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "thinkingmachines/inkling": { "accuracy": 85.507, "latency": 1008.986, "stderr": 2.123, "cost_per_test": 1.357335, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 85.145, "latency": 652.364, "stderr": 2.145, "cost_per_test": 0.44471, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 85.145, "latency": 2246.048, "stderr": 2.145, "cost_per_test": 2.392755, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 85.145, "latency": 3503.03, "stderr": 2.145, "cost_per_test": 17.847147, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 84.42, "latency": 444.684, "stderr": 2.187, "cost_per_test": 2.159387, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 83.696, "latency": 409.394, "stderr": 2.228, "cost_per_test": 0.8503, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 83.333, "latency": 753.94, "stderr": 2.247, "cost_per_test": 3.79389, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 83.333, "latency": 1344.04, "stderr": 2.247, "cost_per_test": 1.499283, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.5": { "accuracy": 82.246, "latency": 669.612, "stderr": 2.304, "cost_per_test": 3.748755, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 82.246, "latency": 3271.126, "stderr": 2.304, "cost_per_test": 23.046646, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 81.944, "latency": 2112.382, "stderr": 2.623, "cost_per_test": 3.206948, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 81.884, "latency": 977.044, "stderr": 2.323, "cost_per_test": 0.192211, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 79.71, "latency": 1057.978, "stderr": 2.425, "cost_per_test": 1.668923, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 78.623, "latency": 724.042, "stderr": 2.472, "cost_per_test": 2.232444, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 78.623, "latency": 1172.084, "stderr": 2.472, "cost_per_test": 8.940461, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 78.313, "latency": 2051.11, "stderr": 3.208, "cost_per_test": 0.415698, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 77.536, "latency": 573.104, "stderr": 2.517, "cost_per_test": 0.12027, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 77.536, "latency": 1009.726, "stderr": 2.517, "cost_per_test": 3.466604, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 76.449, "latency": 186.102, "stderr": 2.559, "cost_per_test": 0.456896, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 76.449, "latency": 809.932, "stderr": 2.559, "cost_per_test": 2.734005, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 75.362, "latency": 394.614, "stderr": 2.598, "cost_per_test": 0.058407, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "stepfun/step-5-preview": { "accuracy": 72.464, "latency": 1301.104, "stderr": 2.694, "cost_per_test": 0.768634, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 72.101, "latency": 154.07, "stderr": 2.705, "cost_per_test": 0.045437, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "zai/glm-5.1": { "accuracy": 71.377, "latency": 526.666, "stderr": 2.726, "cost_per_test": 0.679439, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 70.29, "latency": 2430.912, "stderr": 2.756, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 69.565, "latency": 402.678, "stderr": 2.775, "cost_per_test": 0.039112, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 64.493, "latency": 2490.34, "stderr": 2.886, "cost_per_test": 0.17757, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 64.493, "latency": 4346.874, "stderr": 2.886, "cost_per_test": 6.441935, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 63.768, "latency": 1296.742, "stderr": 2.899, "cost_per_test": 0.809466, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 62.681, "latency": 466.394, "stderr": 2.917, "cost_per_test": 0.546152, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4.3": { "accuracy": 61.594, "latency": 143.374, "stderr": 2.933, "cost_per_test": 0.348081, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 60.145, "latency": 1000.294, "stderr": 2.952, "cost_per_test": 0.706227, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 52.536, "latency": 754.36, "stderr": 3.011, "cost_per_test": 1.066461, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 50.725, "latency": 283.84, "stderr": 3.015, "cost_per_test": 0.174565, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 50.725, "latency": 515.756, "stderr": 3.015, "cost_per_test": 1.014086, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 50.362, "latency": 236.172, "stderr": 3.015, "cost_per_test": 0.244675, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 49.638, "latency": 412.08, "stderr": 3.015, "cost_per_test": 0.42338, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 47.101, "latency": 1341.73, "stderr": 3.01, "cost_per_test": 0.320155, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-m.1": { "accuracy": 38.768, "latency": 324.616, "stderr": 2.938, "cost_per_test": 0.444883, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 38.043, "latency": 550.106, "stderr": 2.928, "cost_per_test": 0.959043, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 26.812, "latency": 116.346, "stderr": 2.671, "cost_per_test": 0.128388, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 26.812, "latency": 3340.984, "stderr": 2.671, "cost_per_test": 0.022637, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 23.913, "latency": 62.2, "stderr": 2.572, "cost_per_test": 0.065573, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "criteria_pass_rate__arbitration_international_dispute_resolution": { "meta/muse_spark_1_3_max": { "accuracy": 96.507, "latency": 931.43, "stderr": 1.216, "cost_per_test": 3.426857, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 96.057, "latency": 1617.612, "stderr": 1.167, "cost_per_test": 2.621978, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 94.265, "latency": 774.504, "stderr": 1.394, "cost_per_test": 0.973952, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 94.265, "latency": 2582.258, "stderr": 1.394, "cost_per_test": 12.524633, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 93.907, "latency": 1037.384, "stderr": 1.435, "cost_per_test": 4.168125, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 93.548, "latency": 961.242, "stderr": 1.473, "cost_per_test": 0.098258, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 92.832, "latency": 2297.508, "stderr": 1.547, "cost_per_test": 14.639679, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.5": { "accuracy": 92.115, "latency": 502.534, "stderr": 1.616, "cost_per_test": 1.759751, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 92.115, "latency": 1522.162, "stderr": 1.616, "cost_per_test": 0.220117, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "fireworks/ember-1": { "accuracy": 92.115, "latency": 2738.558, "stderr": 1.616, "cost_per_test": 5.272515, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 91.756, "latency": 1231.44, "stderr": 1.65, "cost_per_test": 5.708221, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.2": { "accuracy": 91.756, "latency": 1346.446, "stderr": 1.65, "cost_per_test": 2.719674, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3": { "accuracy": 91.756, "latency": 2733.414, "stderr": 1.65, "cost_per_test": 6.270327, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 91.398, "latency": 643.878, "stderr": 1.682, "cost_per_test": 4.788394, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 91.398, "latency": 1715.52, "stderr": 1.682, "cost_per_test": 22.492624, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 91.398, "latency": 2172.626, "stderr": 1.682, "cost_per_test": 4.415408, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 90.681, "latency": 926.49, "stderr": 1.743, "cost_per_test": 0.076998, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 90.323, "latency": 553.876, "stderr": 1.773, "cost_per_test": 0.210863, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 90.323, "latency": 1065.36, "stderr": 1.773, "cost_per_test": 0.22831, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 90.323, "latency": 2729.106, "stderr": 1.773, "cost_per_test": 21.027612, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 89.964, "latency": 4724.642, "stderr": 1.802, "cost_per_test": 3.530728, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 89.606, "latency": 2578.722, "stderr": 1.83, "cost_per_test": 4.266851, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 89.606, "latency": 4238.472, "stderr": 1.83, "cost_per_test": 0.842934, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 89.606, "latency": 14982.956, "stderr": 1.83, "cost_per_test": 93.432713, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 89.247, "latency": 1787.716, "stderr": 1.858, "cost_per_test": 1.332675, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "grok/grok-4.6": { "accuracy": 88.889, "latency": 2088.364, "stderr": 1.885, "cost_per_test": 4.442233, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 88.53, "latency": 2429.578, "stderr": 1.911, "cost_per_test": 5.683671, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 88.172, "latency": 1580.326, "stderr": 1.937, "cost_per_test": 12.077574, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 88.172, "latency": 1678.318, "stderr": 1.937, "cost_per_test": 1.100251, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 87.455, "latency": 1998.744, "stderr": 1.987, "cost_per_test": 11.650358, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 87.097, "latency": 1598.092, "stderr": 2.011, "cost_per_test": 13.097728, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 86.738, "latency": 314.632, "stderr": 2.034, "cost_per_test": 2.222368, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 86.738, "latency": 1866.306, "stderr": 2.034, "cost_per_test": 2.296246, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-6-astra": { "accuracy": 86.022, "latency": 1441.47, "stderr": 2.08, "cost_per_test": 39.13627, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 85.663, "latency": 1067.75, "stderr": 2.102, "cost_per_test": 3.767811, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 83.871, "latency": 828.34, "stderr": 2.206, "cost_per_test": 6.264231, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 83.513, "latency": 395.842, "stderr": 2.226, "cost_per_test": 0.887292, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 82.969, "latency": 3259.595, "stderr": 2.489, "cost_per_test": 22.280508, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 82.796, "latency": 1113.428, "stderr": 2.264, "cost_per_test": 0.504216, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 82.079, "latency": 1371.892, "stderr": 2.3, "cost_per_test": 17.046465, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 81.72, "latency": 676.282, "stderr": 2.318, "cost_per_test": 0.513603, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 80.645, "latency": 386.628, "stderr": 2.37, "cost_per_test": 0.042567, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 78.853, "latency": 854.154, "stderr": 2.449, "cost_per_test": 1.667616, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 78.853, "latency": 1798.27, "stderr": 2.449, "cost_per_test": 1.869712, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 78.495, "latency": 164.272, "stderr": 2.464, "cost_per_test": 0.052176, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 78.495, "latency": 573.844, "stderr": 2.464, "cost_per_test": 2.445749, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 78.495, "latency": 679.778, "stderr": 2.464, "cost_per_test": 7.105465, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 75.986, "latency": 499.18, "stderr": 2.562, "cost_per_test": 0.123822, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 73.835, "latency": 390.77, "stderr": 2.636, "cost_per_test": 0.048701, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 72.76, "latency": 1162.808, "stderr": 2.67, "cost_per_test": 3.307162, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 72.76, "latency": 1694.128, "stderr": 2.67, "cost_per_test": 0.429547, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 70.776, "latency": 5956.935, "stderr": 3.08, "cost_per_test": 8.978748, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.3": { "accuracy": 68.1, "latency": 199.3, "stderr": 2.795, "cost_per_test": 0.354648, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 67.742, "latency": 1801.3, "stderr": 2.804, "cost_per_test": 0.172169, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 66.667, "latency": 940.9, "stderr": 2.827, "cost_per_test": 4.937446, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 63.799, "latency": 226.358, "stderr": 2.882, "cost_per_test": 0.288514, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 63.125, "latency": 799.21, "stderr": 3.826, "cost_per_test": 1.403604, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 62.366, "latency": 1350.512, "stderr": 2.906, "cost_per_test": 37.520705, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 58.423, "latency": 664.974, "stderr": 2.956, "cost_per_test": 0.232631, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 55.556, "latency": 323.41, "stderr": 3.32, "cost_per_test": 0.225776, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 54.122, "latency": 704.482, "stderr": 2.989, "cost_per_test": 0.990325, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 53.405, "latency": 431.252, "stderr": 2.992, "cost_per_test": 0.563167, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 51.971, "latency": 240.988, "stderr": 2.996, "cost_per_test": 0.255694, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 51.613, "latency": 1361.618, "stderr": 2.997, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 50.179, "latency": 1188.18, "stderr": 2.999, "cost_per_test": 0.91661, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 49.462, "latency": 589.422, "stderr": 2.999, "cost_per_test": 0.801309, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 48.746, "latency": 1912.688, "stderr": 2.998, "cost_per_test": 3.164122, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "inception/mercury-2.5": { "accuracy": 48.029, "latency": 151.01, "stderr": 2.996, "cost_per_test": 0.294423, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 43.697, "latency": 1993.245, "stderr": 4.566, "cost_per_test": 0.385748, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-m.1": { "accuracy": 41.921, "latency": 412.718, "stderr": 3.268, "cost_per_test": 0.513055, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 36.918, "latency": 3597.47, "stderr": 2.894, "cost_per_test": 16.01159, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 32.616, "latency": 186.976, "stderr": 2.812, "cost_per_test": 0.346871, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 31.541, "latency": 335.17, "stderr": 2.787, "cost_per_test": 0.370039, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 29.749, "latency": 85.232, "stderr": 2.742, "cost_per_test": 0.079025, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 26.523, "latency": 2837.648, "stderr": 2.648, "cost_per_test": 0.057252, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 22.581, "latency": 644.176, "stderr": 2.508, "cost_per_test": 0.879378, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } }, "criteria_pass_rate__banking_finance": { "meta/muse_spark_1_2": { "accuracy": 97.77, "latency": 1239.682, "stderr": 0.902, "cost_per_test": 1.815007, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 97.398, "latency": 666.63, "stderr": 0.972, "cost_per_test": 2.360297, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 95.911, "latency": 2147.02, "stderr": 1.21, "cost_per_test": 8.507105, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 95.539, "latency": 462.162, "stderr": 1.261, "cost_per_test": 1.30437, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 94.796, "latency": 553.184, "stderr": 1.357, "cost_per_test": 0.664525, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 94.796, "latency": 2610.074, "stderr": 1.357, "cost_per_test": 3.236722, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 94.424, "latency": 694.004, "stderr": 1.402, "cost_per_test": 2.511915, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 94.052, "latency": 698.168, "stderr": 1.445, "cost_per_test": 1.749007, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 92.565, "latency": 1390.374, "stderr": 1.602, "cost_per_test": 15.803642, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 92.565, "latency": 2253.744, "stderr": 1.602, "cost_per_test": 3.51918, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.7": { "accuracy": 92.193, "latency": 2471.01, "stderr": 1.639, "cost_per_test": 8.387325, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 92.193, "latency": 3281.074, "stderr": 1.639, "cost_per_test": 0.617318, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 91.45, "latency": 982.432, "stderr": 1.708, "cost_per_test": 2.744751, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 91.45, "latency": 1770.286, "stderr": 1.708, "cost_per_test": 1.981182, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 91.45, "latency": 2244.842, "stderr": 1.708, "cost_per_test": 3.38767, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 91.078, "latency": 435.288, "stderr": 1.741, "cost_per_test": 2.07755, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 91.078, "latency": 830.14, "stderr": 1.741, "cost_per_test": 0.071494, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 91.078, "latency": 866.12, "stderr": 1.741, "cost_per_test": 0.183967, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 91.078, "latency": 1130.346, "stderr": 1.741, "cost_per_test": 8.75203, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 90.706, "latency": 866.176, "stderr": 1.774, "cost_per_test": 3.089661, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 90.706, "latency": 2139.946, "stderr": 1.774, "cost_per_test": 3.837556, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 90.706, "latency": 3993.99, "stderr": 1.774, "cost_per_test": 35.04233, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 90.335, "latency": 579.452, "stderr": 1.805, "cost_per_test": 3.830095, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 89.963, "latency": 343.366, "stderr": 1.836, "cost_per_test": 0.033503, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 89.963, "latency": 1658.056, "stderr": 1.836, "cost_per_test": 8.036238, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 89.591, "latency": 1258.57, "stderr": 1.865, "cost_per_test": 0.160708, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "stepfun/step-5-preview": { "accuracy": 89.591, "latency": 1611.048, "stderr": 1.865, "cost_per_test": 0.82827, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 89.591, "latency": 3289.418, "stderr": 1.865, "cost_per_test": 16.890636, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 89.219, "latency": 1663.1, "stderr": 1.894, "cost_per_test": 0.858064, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 88.848, "latency": 3006.784, "stderr": 1.923, "cost_per_test": 23.532694, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 88.104, "latency": 1299.434, "stderr": 1.978, "cost_per_test": 20.419711, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/ember-1": { "accuracy": 88.104, "latency": 2922.154, "stderr": 1.978, "cost_per_test": 3.105232, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 87.732, "latency": 737.218, "stderr": 2.004, "cost_per_test": 2.745671, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 87.732, "latency": 1245.96, "stderr": 2.004, "cost_per_test": 9.097239, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 87.361, "latency": 323.646, "stderr": 2.03, "cost_per_test": 0.132613, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 87.361, "latency": 638.052, "stderr": 2.03, "cost_per_test": 0.053598, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 86.617, "latency": 493.128, "stderr": 2.08, "cost_per_test": 0.258578, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 86.617, "latency": 1180.702, "stderr": 2.08, "cost_per_test": 1.170507, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 86.245, "latency": 311.896, "stderr": 2.104, "cost_per_test": 0.609606, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 86.245, "latency": 632.108, "stderr": 2.104, "cost_per_test": 3.453057, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 86.245, "latency": 3388.15, "stderr": 2.104, "cost_per_test": 24.310288, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 85.874, "latency": 715.268, "stderr": 2.128, "cost_per_test": 1.805379, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 85.874, "latency": 848.694, "stderr": 2.128, "cost_per_test": 0.318436, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 85.874, "latency": 1264.306, "stderr": 2.128, "cost_per_test": 2.104294, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 85.874, "latency": 1633.274, "stderr": 2.128, "cost_per_test": 1.681982, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 84.758, "latency": 461.856, "stderr": 2.196, "cost_per_test": 0.095159, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 84.758, "latency": 1314.3, "stderr": 2.196, "cost_per_test": 0.794833, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 84.387, "latency": 784.634, "stderr": 2.217, "cost_per_test": 3.559689, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 84.015, "latency": 708.482, "stderr": 2.239, "cost_per_test": 1.580358, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 84.015, "latency": 2380.224, "stderr": 2.239, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 81.784, "latency": 1895.866, "stderr": 2.358, "cost_per_test": 0.154828, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 81.413, "latency": 1562.144, "stderr": 2.376, "cost_per_test": 10.733228, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 80.669, "latency": 440.778, "stderr": 2.412, "cost_per_test": 1.899233, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 80.297, "latency": 781.298, "stderr": 2.43, "cost_per_test": 2.221578, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 80.297, "latency": 1512.716, "stderr": 2.43, "cost_per_test": 0.334669, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 79.926, "latency": 335.918, "stderr": 2.447, "cost_per_test": 0.353865, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 78.439, "latency": 2267.098, "stderr": 2.512, "cost_per_test": 5.116919, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 74.349, "latency": 132.518, "stderr": 2.668, "cost_per_test": 0.047389, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 73.606, "latency": 297.212, "stderr": 2.692, "cost_per_test": 0.49634, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 72.119, "latency": 1096.08, "stderr": 2.739, "cost_per_test": 1.070964, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5.1": { "accuracy": 71.004, "latency": 413.544, "stderr": 2.772, "cost_per_test": 0.507355, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 69.517, "latency": 371.812, "stderr": 2.812, "cost_per_test": 0.486311, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 68.773, "latency": 206.032, "stderr": 2.831, "cost_per_test": 0.030466, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 68.773, "latency": 2008.814, "stderr": 2.831, "cost_per_test": 3.278736, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 68.401, "latency": 559.192, "stderr": 2.84, "cost_per_test": 0.939858, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 67.286, "latency": 110.462, "stderr": 2.866, "cost_per_test": 0.223979, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 64.312, "latency": 766.152, "stderr": 2.926, "cost_per_test": 0.171268, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 59.48, "latency": 306.848, "stderr": 2.999, "cost_per_test": 0.674698, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 53.636, "latency": 1398.295, "stderr": 4.776, "cost_per_test": 0.219791, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 51.673, "latency": 124.376, "stderr": 3.053, "cost_per_test": 0.220183, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 50.558, "latency": 233.378, "stderr": 3.054, "cost_per_test": 0.356049, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 50.558, "latency": 363.528, "stderr": 3.054, "cost_per_test": 0.694725, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 47.955, "latency": 188.502, "stderr": 3.052, "cost_per_test": 0.160961, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 39.033, "latency": 1283.442, "stderr": 2.98, "cost_per_test": 0.0228, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 36.803, "latency": 189.08, "stderr": 2.946, "cost_per_test": 0.12378, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 36.059, "latency": 49.89, "stderr": 2.933, "cost_per_test": 0.057659, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "criteria_pass_rate__bankruptcy_restructuring": { "meta/muse_spark_1_3": { "accuracy": 95.502, "latency": 909.652, "stderr": 1.221, "cost_per_test": 2.123925, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 93.772, "latency": 1146.54, "stderr": 1.424, "cost_per_test": 1.980681, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 92.734, "latency": 1271.85, "stderr": 1.53, "cost_per_test": 2.982148, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 92.042, "latency": 817.014, "stderr": 1.595, "cost_per_test": 5.003405, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 92.042, "latency": 838.64, "stderr": 1.595, "cost_per_test": 0.878241, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 92.042, "latency": 2433.718, "stderr": 1.595, "cost_per_test": 5.299975, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 91.349, "latency": 2030.7, "stderr": 1.656, "cost_per_test": 10.336011, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 90.311, "latency": 1250.194, "stderr": 1.743, "cost_per_test": 0.309228, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 90.311, "latency": 8020.762, "stderr": 1.743, "cost_per_test": 3.577614, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 89.619, "latency": 2552.876, "stderr": 1.797, "cost_per_test": 5.241005, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/ember-1": { "accuracy": 89.273, "latency": 2664.496, "stderr": 1.823, "cost_per_test": 4.724615, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.5": { "accuracy": 88.235, "latency": 570.796, "stderr": 1.899, "cost_per_test": 2.356863, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 88.235, "latency": 1540.87, "stderr": 1.899, "cost_per_test": 12.196515, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 87.889, "latency": 2251.578, "stderr": 1.922, "cost_per_test": 0.069982, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 87.543, "latency": 2535.254, "stderr": 1.946, "cost_per_test": 4.663981, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 87.197, "latency": 1299.734, "stderr": 1.969, "cost_per_test": 0.116291, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 87.197, "latency": 1962.126, "stderr": 1.969, "cost_per_test": 4.332858, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 87.197, "latency": 2529.162, "stderr": 1.969, "cost_per_test": 13.506458, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 86.851, "latency": 1383.638, "stderr": 1.991, "cost_per_test": 0.186039, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 86.159, "latency": 10274.438, "stderr": 2.035, "cost_per_test": 87.544464, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 85.813, "latency": 1015.696, "stderr": 2.056, "cost_per_test": 5.355597, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 85.813, "latency": 1132.804, "stderr": 2.056, "cost_per_test": 3.545522, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 85.467, "latency": 1782.292, "stderr": 2.077, "cost_per_test": 1.021346, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "tencent/hy4-preview": { "accuracy": 85.467, "latency": 1831.932, "stderr": 2.077, "cost_per_test": 1.145459, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 85.467, "latency": 3939.598, "stderr": 2.077, "cost_per_test": 5.799978, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 84.429, "latency": 1683.26, "stderr": 2.137, "cost_per_test": 23.012162, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 84.083, "latency": 358.702, "stderr": 2.156, "cost_per_test": 3.528761, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 84.083, "latency": 2999.574, "stderr": 2.156, "cost_per_test": 25.189219, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 83.737, "latency": 3302.308, "stderr": 2.175, "cost_per_test": 11.627608, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 83.045, "latency": 1398.952, "stderr": 2.211, "cost_per_test": 11.784326, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 82.007, "latency": 1593.86, "stderr": 2.264, "cost_per_test": 37.668899, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 81.315, "latency": 663.352, "stderr": 2.297, "cost_per_test": 0.549745, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 81.315, "latency": 805.67, "stderr": 2.297, "cost_per_test": 3.843686, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 80.623, "latency": 959.834, "stderr": 2.329, "cost_per_test": 0.310916, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 80.277, "latency": 624.686, "stderr": 2.345, "cost_per_test": 0.211492, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 80.277, "latency": 1149.696, "stderr": 2.345, "cost_per_test": 2.792828, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 80.277, "latency": 1343.498, "stderr": 2.345, "cost_per_test": 1.508844, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 78.201, "latency": 2252.374, "stderr": 2.433, "cost_per_test": 2.436706, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 77.855, "latency": 2176.036, "stderr": 2.447, "cost_per_test": 3.717602, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 76.125, "latency": 377.686, "stderr": 2.512, "cost_per_test": 0.705278, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 76.125, "latency": 738.64, "stderr": 2.512, "cost_per_test": 4.430016, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 75.779, "latency": 301.174, "stderr": 2.525, "cost_per_test": 0.043424, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 75.779, "latency": 494.646, "stderr": 2.525, "cost_per_test": 2.21075, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 75.433, "latency": 331.954, "stderr": 2.537, "cost_per_test": 0.037577, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.1": { "accuracy": 75.433, "latency": 457.39, "stderr": 2.537, "cost_per_test": 0.62914, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 75.087, "latency": 991.99, "stderr": 2.549, "cost_per_test": 3.06916, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 74.394, "latency": 473.106, "stderr": 2.572, "cost_per_test": 0.106362, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.2": { "accuracy": 74.394, "latency": 1531.122, "stderr": 2.572, "cost_per_test": 2.87069, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 73.01, "latency": 1767.562, "stderr": 2.616, "cost_per_test": 0.191614, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 72.664, "latency": 1473.246, "stderr": 2.626, "cost_per_test": 13.525029, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 69.896, "latency": 2768.728, "stderr": 2.703, "cost_per_test": 22.051917, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 68.512, "latency": 940.394, "stderr": 2.737, "cost_per_test": 4.66934, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 62.63, "latency": 218.866, "stderr": 2.851, "cost_per_test": 0.062508, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 61.938, "latency": 1073.03, "stderr": 2.861, "cost_per_test": 0.35608, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 61.592, "latency": 108.948, "stderr": 2.866, "cost_per_test": 0.341968, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 59.17, "latency": 2967.024, "stderr": 2.896, "cost_per_test": 15.129321, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 57.353, "latency": 811.427, "stderr": 3.471, "cost_per_test": 1.062026, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 55.23, "latency": 4087.77, "stderr": 3.223, "cost_per_test": 6.094256, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 55.017, "latency": 461.044, "stderr": 2.931, "cost_per_test": 0.269306, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 54.671, "latency": 897.978, "stderr": 2.933, "cost_per_test": 2.382899, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 54.325, "latency": 1470.566, "stderr": 2.935, "cost_per_test": 1.001911, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.3": { "accuracy": 52.941, "latency": 165.814, "stderr": 2.941, "cost_per_test": 0.462489, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 51.557, "latency": 1356.36, "stderr": 2.945, "cost_per_test": 1.383205, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 51.211, "latency": 480.904, "stderr": 2.945, "cost_per_test": 0.181559, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 46.021, "latency": 5011.65, "stderr": 2.937, "cost_per_test": 0.870454, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 45.675, "latency": 392.87, "stderr": 2.935, "cost_per_test": 0.794801, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "inception/mercury-2.5": { "accuracy": 44.983, "latency": 170.688, "stderr": 2.931, "cost_per_test": 0.234753, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 41.081, "latency": 881.327, "stderr": 3.627, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 35.986, "latency": 476.416, "stderr": 2.828, "cost_per_test": 0.041537, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 30.45, "latency": 252.102, "stderr": 2.712, "cost_per_test": 0.604154, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 30.104, "latency": 913.392, "stderr": 2.703, "cost_per_test": 0.351571, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 25.606, "latency": 217.848, "stderr": 2.572, "cost_per_test": 0.833477, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 20.761, "latency": 265.024, "stderr": 2.39, "cost_per_test": 0.35207, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 19.723, "latency": 200.918, "stderr": 2.345, "cost_per_test": 0.239951, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 16.609, "latency": 53.972, "stderr": 2.193, "cost_per_test": 0.111354, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 10.968, "latency": 202.907, "stderr": 2.518, "cost_per_test": 0.272626, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } }, "criteria_pass_rate__capital_markets": { "meta/muse_spark_1_2": { "accuracy": 96.863, "latency": 960.29, "stderr": 1.094, "cost_per_test": 1.766404, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 96.078, "latency": 707.676, "stderr": 1.218, "cost_per_test": 1.784133, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 95.686, "latency": 2118.234, "stderr": 1.275, "cost_per_test": 9.03284, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 94.51, "latency": 544.732, "stderr": 1.429, "cost_per_test": 1.23096, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 94.51, "latency": 586.44, "stderr": 1.429, "cost_per_test": 2.819389, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 94.51, "latency": 1082.806, "stderr": 1.429, "cost_per_test": 2.051823, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 94.118, "latency": 2226.112, "stderr": 1.476, "cost_per_test": 3.540513, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 93.725, "latency": 499.572, "stderr": 1.522, "cost_per_test": 3.218847, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 93.725, "latency": 2231.008, "stderr": 1.522, "cost_per_test": 3.463263, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 92.941, "latency": 2502.272, "stderr": 1.607, "cost_per_test": 9.722399, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 92.549, "latency": 557.858, "stderr": 1.648, "cost_per_test": 0.645246, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 92.157, "latency": 634.18, "stderr": 1.687, "cost_per_test": 2.399124, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 92.157, "latency": 1231.11, "stderr": 1.687, "cost_per_test": 9.299876, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 92.157, "latency": 1236.618, "stderr": 1.687, "cost_per_test": 0.197788, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 92.157, "latency": 1347.776, "stderr": 1.687, "cost_per_test": 2.151201, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 92.157, "latency": 1604.092, "stderr": 1.687, "cost_per_test": 0.917971, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 92.157, "latency": 2665.926, "stderr": 1.687, "cost_per_test": 32.845013, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 91.765, "latency": 963.78, "stderr": 1.725, "cost_per_test": 1.900859, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 91.765, "latency": 2210.02, "stderr": 1.725, "cost_per_test": 0.433283, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 91.373, "latency": 1084.768, "stderr": 1.762, "cost_per_test": 17.303719, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 91.373, "latency": 1088.286, "stderr": 1.762, "cost_per_test": 0.136395, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 91.373, "latency": 1359.526, "stderr": 1.762, "cost_per_test": 17.509413, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 91.373, "latency": 1460.304, "stderr": 1.762, "cost_per_test": 0.791921, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3": { "accuracy": 91.373, "latency": 1692.948, "stderr": 1.762, "cost_per_test": 3.089995, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 90.98, "latency": 1121.472, "stderr": 1.797, "cost_per_test": 0.048956, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 90.98, "latency": 2856.812, "stderr": 1.797, "cost_per_test": 14.684627, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 90.196, "latency": 1907.572, "stderr": 1.866, "cost_per_test": 2.375674, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 90.196, "latency": 2863.46, "stderr": 1.866, "cost_per_test": 20.393428, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 89.804, "latency": 2332.704, "stderr": 1.899, "cost_per_test": 1.737948, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 89.412, "latency": 322.07, "stderr": 1.931, "cost_per_test": 2.854243, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 89.412, "latency": 766.514, "stderr": 1.931, "cost_per_test": 0.073201, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 89.412, "latency": 788.504, "stderr": 1.931, "cost_per_test": 0.197475, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 89.02, "latency": 931.196, "stderr": 1.962, "cost_per_test": 0.941373, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 89.02, "latency": 1292.068, "stderr": 1.962, "cost_per_test": 5.128468, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 88.235, "latency": 456.474, "stderr": 2.022, "cost_per_test": 0.16244, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 88.235, "latency": 1228.556, "stderr": 2.022, "cost_per_test": 10.574556, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 88.235, "latency": 1787.54, "stderr": 2.022, "cost_per_test": 3.174123, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 87.451, "latency": 721.686, "stderr": 2.079, "cost_per_test": 2.678383, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 87.451, "latency": 2330.642, "stderr": 2.079, "cost_per_test": 18.356578, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 87.059, "latency": 507.738, "stderr": 2.106, "cost_per_test": 0.283023, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 87.059, "latency": 1055.51, "stderr": 2.106, "cost_per_test": 1.763195, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 86.667, "latency": 740.832, "stderr": 2.133, "cost_per_test": 2.617447, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 86.667, "latency": 923.596, "stderr": 2.133, "cost_per_test": 2.396947, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 86.275, "latency": 350.396, "stderr": 2.159, "cost_per_test": 0.764042, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 84.706, "latency": 1598.592, "stderr": 2.258, "cost_per_test": 1.415374, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 83.922, "latency": 692.736, "stderr": 2.305, "cost_per_test": 1.167551, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 83.137, "latency": 1453.458, "stderr": 2.349, "cost_per_test": 0.150847, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 82.745, "latency": 413.782, "stderr": 2.371, "cost_per_test": 1.728373, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 82.353, "latency": 250.502, "stderr": 2.392, "cost_per_test": 0.031194, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 81.569, "latency": 678.396, "stderr": 2.433, "cost_per_test": 3.277627, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 80.0, "latency": 5620.726, "stderr": 2.51, "cost_per_test": 4.981585, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 79.216, "latency": 374.008, "stderr": 2.546, "cost_per_test": 0.031216, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 77.647, "latency": 97.31, "stderr": 2.614, "cost_per_test": 0.251837, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 74.902, "latency": 122.554, "stderr": 2.721, "cost_per_test": 0.036275, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "grok/grok-4.3": { "accuracy": 74.902, "latency": 125.772, "stderr": 2.721, "cost_per_test": 0.232911, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 73.725, "latency": 578.198, "stderr": 2.762, "cost_per_test": 3.800605, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 73.125, "latency": 1403.207, "stderr": 3.516, "cost_per_test": 0.284413, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 72.941, "latency": 255.418, "stderr": 2.788, "cost_per_test": 0.119538, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 72.157, "latency": 1100.542, "stderr": 2.812, "cost_per_test": 0.64382, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.1": { "accuracy": 71.765, "latency": 393.834, "stderr": 2.824, "cost_per_test": 0.443784, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 71.765, "latency": 688.528, "stderr": 2.824, "cost_per_test": 1.181787, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 71.373, "latency": 1245.51, "stderr": 2.836, "cost_per_test": 5.036098, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 70.588, "latency": 397.338, "stderr": 2.859, "cost_per_test": 0.490152, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 69.804, "latency": 257.284, "stderr": 2.881, "cost_per_test": 0.735522, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 68.627, "latency": 443.016, "stderr": 2.911, "cost_per_test": 0.110985, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 67.143, "latency": 626.825, "stderr": 3.249, "cost_per_test": 0.755231, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 65.714, "latency": 347.665, "stderr": 3.283, "cost_per_test": 0.13166, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 64.314, "latency": 188.528, "stderr": 3.006, "cost_per_test": 0.261643, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 56.078, "latency": 327.828, "stderr": 3.114, "cost_per_test": 0.415094, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 54.902, "latency": 975.162, "stderr": 3.122, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 54.51, "latency": 962.008, "stderr": 3.124, "cost_per_test": 0.329241, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 50.588, "latency": 263.832, "stderr": 3.137, "cost_per_test": 0.291257, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 40.0, "latency": 331.896, "stderr": 3.074, "cost_per_test": 0.366377, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 38.824, "latency": 35.432, "stderr": 3.058, "cost_per_test": 0.043334, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 32.157, "latency": 950.504, "stderr": 2.931, "cost_per_test": 0.018306, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-2.5": { "accuracy": 26.667, "latency": 64.014, "stderr": 2.775, "cost_per_test": 0.121463, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null } }, "criteria_pass_rate__corporate_governance": { "meta/muse_spark_1_3_max": { "accuracy": 97.26, "latency": 1143.856, "stderr": 0.957, "cost_per_test": 2.672757, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 97.26, "latency": 2442.838, "stderr": 0.957, "cost_per_test": 7.288851, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 96.918, "latency": 1321.064, "stderr": 1.013, "cost_per_test": 1.030941, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 96.233, "latency": 1426.99, "stderr": 1.116, "cost_per_test": 2.15945, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 95.548, "latency": 874.242, "stderr": 1.209, "cost_per_test": 2.728836, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 94.178, "latency": 736.924, "stderr": 1.373, "cost_per_test": 0.07785, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 93.493, "latency": 543.246, "stderr": 1.446, "cost_per_test": 3.202037, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.2": { "accuracy": 93.151, "latency": 1380.374, "stderr": 1.481, "cost_per_test": 2.394515, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 93.151, "latency": 1552.306, "stderr": 1.481, "cost_per_test": 4.414299, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.6": { "accuracy": 93.151, "latency": 2166.744, "stderr": 1.481, "cost_per_test": 5.377027, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 92.808, "latency": 935.4, "stderr": 1.514, "cost_per_test": 0.202579, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 92.808, "latency": 1879.302, "stderr": 1.514, "cost_per_test": 0.069229, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 92.808, "latency": 2334.13, "stderr": 1.514, "cost_per_test": 5.247151, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 92.123, "latency": 1644.848, "stderr": 1.579, "cost_per_test": 18.661957, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 92.123, "latency": 2376.882, "stderr": 1.579, "cost_per_test": 3.186137, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.7": { "accuracy": 91.781, "latency": 2513.988, "stderr": 1.61, "cost_per_test": 10.269134, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 91.781, "latency": 3315.052, "stderr": 1.61, "cost_per_test": 4.21486, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 91.781, "latency": 3929.774, "stderr": 1.61, "cost_per_test": 2.441136, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 91.438, "latency": 676.244, "stderr": 1.64, "cost_per_test": 0.182687, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.5": { "accuracy": 90.753, "latency": 669.87, "stderr": 1.698, "cost_per_test": 1.86509, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 90.411, "latency": 329.304, "stderr": 1.726, "cost_per_test": 3.011145, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "stepfun/step-5-preview": { "accuracy": 89.384, "latency": 1546.64, "stderr": 1.806, "cost_per_test": 0.874361, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 89.384, "latency": 1554.822, "stderr": 1.806, "cost_per_test": 9.763798, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 89.384, "latency": 1958.866, "stderr": 1.806, "cost_per_test": 10.888301, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 89.041, "latency": 1412.708, "stderr": 1.831, "cost_per_test": 11.140007, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 88.699, "latency": 1740.68, "stderr": 1.856, "cost_per_test": 1.030237, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-astra": { "accuracy": 88.356, "latency": 1583.514, "stderr": 1.88, "cost_per_test": 33.095665, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 88.014, "latency": 372.972, "stderr": 1.904, "cost_per_test": 0.04276, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 88.014, "latency": 1402.604, "stderr": 1.904, "cost_per_test": 1.37422, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 88.014, "latency": 1409.04, "stderr": 1.904, "cost_per_test": 0.174308, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 88.014, "latency": 2434.85, "stderr": 1.904, "cost_per_test": 3.975993, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 88.014, "latency": 2832.07, "stderr": 1.904, "cost_per_test": 4.666701, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 86.986, "latency": 1311.706, "stderr": 1.972, "cost_per_test": 0.826881, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 86.986, "latency": 1323.154, "stderr": 1.972, "cost_per_test": 3.276748, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 86.644, "latency": 693.504, "stderr": 1.994, "cost_per_test": 0.327343, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 86.644, "latency": 812.582, "stderr": 1.994, "cost_per_test": 1.792585, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 86.644, "latency": 10823.312, "stderr": 1.994, "cost_per_test": 61.982943, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 85.959, "latency": 355.344, "stderr": 2.037, "cost_per_test": 0.585215, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 85.959, "latency": 1980.132, "stderr": 2.037, "cost_per_test": 1.221399, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 85.959, "latency": 2145.662, "stderr": 2.037, "cost_per_test": 3.673218, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 85.274, "latency": 518.074, "stderr": 2.077, "cost_per_test": 2.188345, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 84.932, "latency": 386.21, "stderr": 2.097, "cost_per_test": 0.034168, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 84.589, "latency": 854.396, "stderr": 2.117, "cost_per_test": 0.246185, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 84.589, "latency": 1606.784, "stderr": 2.117, "cost_per_test": 1.493244, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 83.219, "latency": 3550.886, "stderr": 2.191, "cost_per_test": 23.927034, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 81.507, "latency": 937.754, "stderr": 2.276, "cost_per_test": 1.394328, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-sol": { "accuracy": 81.164, "latency": 754.62, "stderr": 2.292, "cost_per_test": 4.210018, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 80.822, "latency": 617.67, "stderr": 2.308, "cost_per_test": 0.095359, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 79.11, "latency": 161.014, "stderr": 2.383, "cost_per_test": 0.042448, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.5": { "accuracy": 78.425, "latency": 757.952, "stderr": 2.411, "cost_per_test": 6.425704, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 76.027, "latency": 1194.222, "stderr": 2.503, "cost_per_test": 0.37409, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 75.685, "latency": 1349.544, "stderr": 2.515, "cost_per_test": 10.906051, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 75.342, "latency": 139.304, "stderr": 2.527, "cost_per_test": 0.392709, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 75.0, "latency": 174.342, "stderr": 2.538, "cost_per_test": 0.394478, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 74.658, "latency": 465.482, "stderr": 2.55, "cost_per_test": 0.593195, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 73.288, "latency": 895.086, "stderr": 2.594, "cost_per_test": 3.705633, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 72.945, "latency": 464.67, "stderr": 2.604, "cost_per_test": 0.55015, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 72.945, "latency": 807.578, "stderr": 2.604, "cost_per_test": 3.489705, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 72.945, "latency": 1552.808, "stderr": 2.604, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 71.575, "latency": 4249.13, "stderr": 2.644, "cost_per_test": 0.906277, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 70.89, "latency": 3385.064, "stderr": 2.663, "cost_per_test": 24.770063, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 65.753, "latency": 1688.058, "stderr": 2.782, "cost_per_test": 0.159111, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 64.384, "latency": 364.926, "stderr": 2.807, "cost_per_test": 0.133065, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 64.041, "latency": 437.184, "stderr": 2.813, "cost_per_test": 0.458097, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 62.671, "latency": 4642.184, "stderr": 2.835, "cost_per_test": 6.850674, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 61.301, "latency": 422.362, "stderr": 2.855, "cost_per_test": 0.495369, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 60.616, "latency": 445.236, "stderr": 2.864, "cost_per_test": 0.63414, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 57.877, "latency": 166.626, "stderr": 2.894, "cost_per_test": 0.171496, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 57.642, "latency": 432.855, "stderr": 3.272, "cost_per_test": 0.156264, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 57.192, "latency": 298.072, "stderr": 2.901, "cost_per_test": 1.357341, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 51.37, "latency": 960.338, "stderr": 2.93, "cost_per_test": 0.033144, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 51.092, "latency": 506.593, "stderr": 3.311, "cost_per_test": 0.786027, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 50.685, "latency": 3464.234, "stderr": 2.931, "cost_per_test": 16.225332, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "inception/mercury-2.5": { "accuracy": 43.836, "latency": 105.77, "stderr": 2.909, "cost_per_test": 0.271182, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 43.333, "latency": 2324.97, "stderr": 6.451, "cost_per_test": 0.445506, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 29.795, "latency": 45.132, "stderr": 2.681, "cost_per_test": 0.053496, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "criteria_pass_rate__corporate_ma": { "meta/muse_spark_1_1": { "accuracy": 96.154, "latency": 778.878, "stderr": 1.09, "cost_per_test": 0.847903, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 96.154, "latency": 949.974, "stderr": 1.09, "cost_per_test": 2.35844, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 96.154, "latency": 2234.264, "stderr": 1.09, "cost_per_test": 8.132612, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 95.833, "latency": 1265.256, "stderr": 1.133, "cost_per_test": 1.735278, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.5": { "accuracy": 95.192, "latency": 767.346, "stderr": 1.213, "cost_per_test": 2.416866, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.6": { "accuracy": 95.192, "latency": 3381.568, "stderr": 1.213, "cost_per_test": 4.118716, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 94.872, "latency": 782.066, "stderr": 1.251, "cost_per_test": 1.932419, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 94.872, "latency": 1150.196, "stderr": 1.251, "cost_per_test": 0.07413, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 94.551, "latency": 2112.8, "stderr": 1.287, "cost_per_test": 2.209117, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 94.551, "latency": 2879.192, "stderr": 1.287, "cost_per_test": 3.937041, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 94.231, "latency": 597.146, "stderr": 1.322, "cost_per_test": 0.050437, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 93.91, "latency": 1632.694, "stderr": 1.356, "cost_per_test": 0.94797, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 93.91, "latency": 2302.844, "stderr": 1.356, "cost_per_test": 3.938856, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/ember-1": { "accuracy": 93.91, "latency": 2451.596, "stderr": 1.356, "cost_per_test": 3.296217, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 93.91, "latency": 3391.348, "stderr": 1.356, "cost_per_test": 25.998286, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 93.59, "latency": 531.144, "stderr": 1.389, "cost_per_test": 3.283063, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 93.59, "latency": 2461.11, "stderr": 1.389, "cost_per_test": 0.176755, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 93.59, "latency": 2657.59, "stderr": 1.389, "cost_per_test": 13.092829, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 93.59, "latency": 4606.318, "stderr": 1.389, "cost_per_test": 3.767001, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 93.269, "latency": 1344.032, "stderr": 1.421, "cost_per_test": 10.138066, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 93.269, "latency": 1454.082, "stderr": 1.421, "cost_per_test": 0.778644, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 92.949, "latency": 1524.986, "stderr": 1.452, "cost_per_test": 10.04374, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 92.949, "latency": 2112.592, "stderr": 1.452, "cost_per_test": 3.457385, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 92.949, "latency": 3062.552, "stderr": 1.452, "cost_per_test": 4.419748, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 92.308, "latency": 305.354, "stderr": 1.511, "cost_per_test": 2.25119, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 92.308, "latency": 1088.096, "stderr": 1.511, "cost_per_test": 0.144414, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 92.308, "latency": 3182.286, "stderr": 1.511, "cost_per_test": 24.674742, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 91.987, "latency": 444.366, "stderr": 1.539, "cost_per_test": 0.166084, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 91.987, "latency": 1471.018, "stderr": 1.539, "cost_per_test": 18.410087, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 91.667, "latency": 2821.808, "stderr": 1.567, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 91.346, "latency": 799.498, "stderr": 1.594, "cost_per_test": 3.550379, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 91.026, "latency": 1143.366, "stderr": 1.621, "cost_per_test": 3.051018, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 91.026, "latency": 1282.05, "stderr": 1.621, "cost_per_test": 0.73839, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 90.705, "latency": 1816.776, "stderr": 1.646, "cost_per_test": 28.643676, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 90.385, "latency": 2906.61, "stderr": 1.672, "cost_per_test": 0.512019, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 90.064, "latency": 748.426, "stderr": 1.696, "cost_per_test": 1.599282, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 90.064, "latency": 1138.958, "stderr": 1.696, "cost_per_test": 1.100807, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-6-sol": { "accuracy": 89.744, "latency": 894.128, "stderr": 1.72, "cost_per_test": 2.742624, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 89.744, "latency": 3609.028, "stderr": 1.72, "cost_per_test": 38.53423, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 89.103, "latency": 1091.8, "stderr": 1.767, "cost_per_test": 0.332689, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 88.782, "latency": 1496.27, "stderr": 1.79, "cost_per_test": 1.496984, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 88.462, "latency": 451.45, "stderr": 1.812, "cost_per_test": 0.855036, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 88.462, "latency": 2809.288, "stderr": 1.812, "cost_per_test": 7.318404, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 87.5, "latency": 458.588, "stderr": 1.875, "cost_per_test": 0.05375, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.5": { "accuracy": 87.5, "latency": 803.612, "stderr": 1.875, "cost_per_test": 3.395472, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 87.5, "latency": 832.006, "stderr": 1.875, "cost_per_test": 2.104291, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 87.179, "latency": 685.316, "stderr": 1.896, "cost_per_test": 0.396871, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 86.859, "latency": 360.64, "stderr": 1.916, "cost_per_test": 0.035857, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 86.859, "latency": 1028.182, "stderr": 1.916, "cost_per_test": 3.067955, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 86.538, "latency": 421.036, "stderr": 1.935, "cost_per_test": 1.877373, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.1": { "accuracy": 85.897, "latency": 499.4, "stderr": 1.974, "cost_per_test": 0.580534, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 85.577, "latency": 1261.026, "stderr": 1.992, "cost_per_test": 12.611383, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 83.974, "latency": 505.846, "stderr": 2.08, "cost_per_test": 0.104926, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 82.051, "latency": 375.644, "stderr": 2.176, "cost_per_test": 0.438626, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 81.731, "latency": 901.91, "stderr": 2.191, "cost_per_test": 2.075068, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 80.769, "latency": 1415.18, "stderr": 2.235, "cost_per_test": 0.362556, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 79.487, "latency": 1975.746, "stderr": 2.29, "cost_per_test": 0.20359, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 78.846, "latency": 130.574, "stderr": 2.316, "cost_per_test": 0.042211, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "zai/glm-5.2": { "accuracy": 78.846, "latency": 1002.308, "stderr": 2.316, "cost_per_test": 1.650999, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 77.244, "latency": 248.88, "stderr": 2.377, "cost_per_test": 0.330445, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 75.962, "latency": 3174.424, "stderr": 2.423, "cost_per_test": 15.021633, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 72.84, "latency": 804.433, "stderr": 3.505, "cost_per_test": 1.456426, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 70.192, "latency": 239.62, "stderr": 2.594, "cost_per_test": 0.475678, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 61.859, "latency": 412.996, "stderr": 2.754, "cost_per_test": 0.25487, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 61.218, "latency": 457.828, "stderr": 2.763, "cost_per_test": 0.17506, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 59.615, "latency": 189.98, "stderr": 2.782, "cost_per_test": 0.464081, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 58.974, "latency": 499.322, "stderr": 2.789, "cost_per_test": 0.719898, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "thinkingmachines/inkling": { "accuracy": 58.654, "latency": 922.502, "stderr": 2.792, "cost_per_test": 1.157097, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 54.487, "latency": 4108.97, "stderr": 2.824, "cost_per_test": 6.680874, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 52.564, "latency": 316.128, "stderr": 2.832, "cost_per_test": 0.414307, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "inception/mercury-2.5": { "accuracy": 51.282, "latency": 97.04, "stderr": 2.834, "cost_per_test": 0.197043, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 41.987, "latency": 134.364, "stderr": 2.799, "cost_per_test": 0.246824, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 38.462, "latency": 871.134, "stderr": 2.759, "cost_per_test": 0.234252, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 36.538, "latency": 198.466, "stderr": 2.731, "cost_per_test": 0.9394, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 33.333, "latency": 1198.086, "stderr": 2.673, "cost_per_test": 0.027002, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 32.372, "latency": 68.164, "stderr": 2.653, "cost_per_test": 0.091726, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "criteria_pass_rate__data_privacy_cybersecurity": { "meta/muse_spark_1_3_max": { "accuracy": 99.293, "latency": 873.068, "stderr": 0.499, "cost_per_test": 1.819537, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 97.88, "latency": 4005.678, "stderr": 0.858, "cost_per_test": 4.387069, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 97.527, "latency": 2394.758, "stderr": 0.925, "cost_per_test": 8.325019, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3": { "accuracy": 97.527, "latency": 2679.258, "stderr": 0.925, "cost_per_test": 4.796128, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 96.82, "latency": 735.874, "stderr": 1.045, "cost_per_test": 0.083103, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 96.82, "latency": 3410.766, "stderr": 1.045, "cost_per_test": 35.475522, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 96.466, "latency": 1141.602, "stderr": 1.099, "cost_per_test": 3.424472, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "stepfun/step-5-preview": { "accuracy": 96.104, "latency": 1997.14, "stderr": 1.276, "cost_per_test": 1.406748, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 95.76, "latency": 1731.114, "stderr": 1.2, "cost_per_test": 2.741491, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 95.406, "latency": 2251.364, "stderr": 1.247, "cost_per_test": 2.805536, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 95.406, "latency": 2512.86, "stderr": 1.247, "cost_per_test": 5.085316, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.5": { "accuracy": 95.053, "latency": 535.282, "stderr": 1.291, "cost_per_test": 1.84691, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 95.053, "latency": 554.804, "stderr": 1.291, "cost_per_test": 3.802232, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.2": { "accuracy": 95.053, "latency": 1354.47, "stderr": 1.291, "cost_per_test": 2.842591, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 95.053, "latency": 1783.684, "stderr": 1.291, "cost_per_test": 1.156317, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "fireworks/ember-1": { "accuracy": 95.053, "latency": 2182.8, "stderr": 1.291, "cost_per_test": 3.377518, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 94.7, "latency": 795.424, "stderr": 1.334, "cost_per_test": 0.204203, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 94.7, "latency": 821.752, "stderr": 1.334, "cost_per_test": 0.067318, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 94.7, "latency": 948.004, "stderr": 1.334, "cost_per_test": 4.57258, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 94.346, "latency": 1370.184, "stderr": 1.375, "cost_per_test": 11.031307, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 94.346, "latency": 1595.642, "stderr": 1.375, "cost_per_test": 20.041934, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 94.346, "latency": 2203.2, "stderr": 1.375, "cost_per_test": 0.404976, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 94.346, "latency": 4101.838, "stderr": 1.375, "cost_per_test": 31.449565, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 93.993, "latency": 561.456, "stderr": 1.415, "cost_per_test": 0.178414, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 93.694, "latency": 654.12, "stderr": 1.635, "cost_per_test": 0.83785, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 93.286, "latency": 302.728, "stderr": 1.49, "cost_per_test": 2.149328, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 93.286, "latency": 682.68, "stderr": 1.49, "cost_per_test": 2.439638, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 93.286, "latency": 1553.434, "stderr": 1.49, "cost_per_test": 28.026206, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 92.933, "latency": 1108.122, "stderr": 1.526, "cost_per_test": 0.161869, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.5": { "accuracy": 92.226, "latency": 641.514, "stderr": 1.594, "cost_per_test": 3.983816, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 92.226, "latency": 1542.712, "stderr": 1.594, "cost_per_test": 2.157888, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 91.873, "latency": 1506.148, "stderr": 1.627, "cost_per_test": 12.882643, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 91.519, "latency": 725.044, "stderr": 1.659, "cost_per_test": 0.400108, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 91.519, "latency": 1096.256, "stderr": 1.659, "cost_per_test": 1.875677, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 91.166, "latency": 1857.564, "stderr": 1.69, "cost_per_test": 9.396413, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 90.106, "latency": 825.86, "stderr": 1.778, "cost_per_test": 3.748145, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 90.106, "latency": 1177.988, "stderr": 1.778, "cost_per_test": 9.289774, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 89.753, "latency": 2246.312, "stderr": 1.806, "cost_per_test": 3.575179, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 89.046, "latency": 1034.436, "stderr": 1.86, "cost_per_test": 3.1184, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 88.693, "latency": 560.682, "stderr": 1.886, "cost_per_test": 0.628345, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 88.339, "latency": 1847.046, "stderr": 1.911, "cost_per_test": 1.368981, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 87.633, "latency": 1345.862, "stderr": 1.96, "cost_per_test": 3.070707, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 86.926, "latency": 538.888, "stderr": 2.008, "cost_per_test": 0.100374, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 85.866, "latency": 344.946, "stderr": 2.075, "cost_per_test": 0.046308, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 85.866, "latency": 375.346, "stderr": 2.075, "cost_per_test": 0.741361, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 85.159, "latency": 904.912, "stderr": 2.117, "cost_per_test": 3.103135, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-luna": { "accuracy": 85.159, "latency": 1442.554, "stderr": 2.117, "cost_per_test": 0.343032, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 84.452, "latency": 460.386, "stderr": 2.158, "cost_per_test": 1.788615, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 83.746, "latency": 2115.014, "stderr": 2.197, "cost_per_test": 9.177084, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 81.625, "latency": 1920.932, "stderr": 2.306, "cost_per_test": 0.178357, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 79.859, "latency": 407.004, "stderr": 2.388, "cost_per_test": 0.039793, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 79.152, "latency": 812.514, "stderr": 2.419, "cost_per_test": 2.616626, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 77.385, "latency": 161.092, "stderr": 2.491, "cost_per_test": 0.043571, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 77.032, "latency": 3526.54, "stderr": 2.505, "cost_per_test": 16.205227, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 74.558, "latency": 985.884, "stderr": 2.594, "cost_per_test": 0.697975, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.3": { "accuracy": 73.852, "latency": 159.248, "stderr": 2.617, "cost_per_test": 0.356562, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 73.852, "latency": 2891.984, "stderr": 2.617, "cost_per_test": 21.050628, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 72.792, "latency": 952.242, "stderr": 2.65, "cost_per_test": 1.636783, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 72.438, "latency": 298.162, "stderr": 2.661, "cost_per_test": 0.320555, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 71.731, "latency": 1389.888, "stderr": 2.682, "cost_per_test": 0.392959, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 70.13, "latency": 2722.318, "stderr": 3.018, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 69.611, "latency": 1871.402, "stderr": 2.739, "cost_per_test": 3.227633, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 67.138, "latency": 2145.894, "stderr": 2.797, "cost_per_test": 6.673207, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 63.958, "latency": 329.54, "stderr": 2.859, "cost_per_test": 0.12102, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 59.364, "latency": 278.552, "stderr": 2.925, "cost_per_test": 0.375904, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 58.009, "latency": 463.488, "stderr": 3.254, "cost_per_test": 0.731036, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 56.537, "latency": 389.142, "stderr": 2.952, "cost_per_test": 1.359549, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 52.193, "latency": 205.34, "stderr": 3.315, "cost_per_test": 0.180563, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 51.943, "latency": 521.026, "stderr": 2.975, "cost_per_test": 0.572555, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "inception/mercury-2.5": { "accuracy": 47.35, "latency": 77.594, "stderr": 2.973, "cost_per_test": 0.14917, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 46.996, "latency": 379.96, "stderr": 2.972, "cost_per_test": 1.020644, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 46.729, "latency": 2031.03, "stderr": 4.846, "cost_per_test": 0.387754, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 44.523, "latency": 338.276, "stderr": 2.96, "cost_per_test": 0.469152, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 40.989, "latency": 127.518, "stderr": 2.929, "cost_per_test": 0.195281, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 37.809, "latency": 2338.224, "stderr": 2.888, "cost_per_test": 0.014473, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 29.682, "latency": 78.586, "stderr": 2.721, "cost_per_test": 0.057493, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "criteria_pass_rate__emerging_companies_venture_capital": { "meta/muse_spark_1_3_max": { "accuracy": 95.904, "latency": 749.636, "stderr": 1.16, "cost_per_test": 2.002735, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 95.563, "latency": 976.982, "stderr": 1.205, "cost_per_test": 1.968528, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 95.222, "latency": 1654.386, "stderr": 1.248, "cost_per_test": 1.884283, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 94.881, "latency": 3078.254, "stderr": 1.29, "cost_per_test": 13.959833, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 93.86, "latency": 1407.428, "stderr": 1.593, "cost_per_test": 0.222921, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 93.857, "latency": 1722.228, "stderr": 1.405, "cost_per_test": 17.394862, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 93.515, "latency": 1857.83, "stderr": 1.441, "cost_per_test": 0.956726, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "grok/grok-4.7": { "accuracy": 92.833, "latency": 2941.5, "stderr": 1.51, "cost_per_test": 13.442724, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 92.15, "latency": 2745.84, "stderr": 1.574, "cost_per_test": 3.610617, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3": { "accuracy": 92.15, "latency": 3656.866, "stderr": 1.574, "cost_per_test": 4.908979, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 91.809, "latency": 798.338, "stderr": 1.605, "cost_per_test": 0.754855, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 91.809, "latency": 2459.848, "stderr": 1.605, "cost_per_test": 4.683139, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 91.468, "latency": 1995.764, "stderr": 1.635, "cost_per_test": 0.990763, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 91.468, "latency": 8097.042, "stderr": 1.635, "cost_per_test": 54.106639, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 90.785, "latency": 2170.442, "stderr": 1.693, "cost_per_test": 2.192372, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 90.444, "latency": 830.042, "stderr": 1.72, "cost_per_test": 0.060188, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 90.444, "latency": 886.134, "stderr": 1.72, "cost_per_test": 0.078418, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 90.444, "latency": 1442.672, "stderr": 1.72, "cost_per_test": 0.175319, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 90.444, "latency": 2882.79, "stderr": 1.72, "cost_per_test": 4.18071, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/ember-1": { "accuracy": 90.102, "latency": 3294.328, "stderr": 1.748, "cost_per_test": 3.003809, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 89.761, "latency": 1055.05, "stderr": 1.774, "cost_per_test": 3.941127, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 89.761, "latency": 3218.206, "stderr": 1.774, "cost_per_test": 3.954288, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 89.078, "latency": 1499.998, "stderr": 1.825, "cost_per_test": 8.692762, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 89.078, "latency": 3143.702, "stderr": 1.825, "cost_per_test": 3.981076, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 89.078, "latency": 3356.11, "stderr": 1.825, "cost_per_test": 21.32291, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 88.737, "latency": 799.56, "stderr": 1.85, "cost_per_test": 2.317085, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 88.737, "latency": 1318.76, "stderr": 1.85, "cost_per_test": 20.549812, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 88.737, "latency": 2237.394, "stderr": 1.85, "cost_per_test": 3.691787, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 88.737, "latency": 3341.218, "stderr": 1.85, "cost_per_test": 16.606099, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 87.713, "latency": 628.536, "stderr": 1.921, "cost_per_test": 0.17285, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 87.372, "latency": 415.078, "stderr": 1.944, "cost_per_test": 0.76098, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.2": { "accuracy": 87.372, "latency": 1326.42, "stderr": 1.944, "cost_per_test": 2.290803, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 87.372, "latency": 1784.084, "stderr": 1.944, "cost_per_test": 12.825865, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 87.031, "latency": 780.25, "stderr": 1.966, "cost_per_test": 0.39444, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 86.689, "latency": 406.88, "stderr": 1.988, "cost_per_test": 2.01759, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 86.348, "latency": 659.712, "stderr": 2.009, "cost_per_test": 2.105221, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 86.348, "latency": 1276.908, "stderr": 2.009, "cost_per_test": 0.378632, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 85.666, "latency": 3176.12, "stderr": 2.051, "cost_per_test": 9.68991, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 85.324, "latency": 1289.842, "stderr": 2.071, "cost_per_test": 10.388772, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 85.324, "latency": 1939.344, "stderr": 2.071, "cost_per_test": 1.696531, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 84.983, "latency": 1109.156, "stderr": 2.091, "cost_per_test": 3.23361, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 83.959, "latency": 1159.888, "stderr": 2.148, "cost_per_test": 1.881256, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 83.276, "latency": 799.138, "stderr": 2.184, "cost_per_test": 1.664943, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 83.276, "latency": 1607.58, "stderr": 2.184, "cost_per_test": 1.729985, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 82.935, "latency": 947.998, "stderr": 2.202, "cost_per_test": 3.671658, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 82.594, "latency": 573.73, "stderr": 2.219, "cost_per_test": 0.105559, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 81.57, "latency": 1316.004, "stderr": 2.269, "cost_per_test": 0.769426, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 78.498, "latency": 481.848, "stderr": 2.404, "cost_per_test": 2.122257, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 77.816, "latency": 3672.84, "stderr": 2.431, "cost_per_test": 0.551792, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 75.427, "latency": 356.59, "stderr": 2.519, "cost_per_test": 0.047376, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 73.379, "latency": 2022.664, "stderr": 2.586, "cost_per_test": 0.18381, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 72.355, "latency": 177.306, "stderr": 2.617, "cost_per_test": 0.051943, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 70.815, "latency": 784.232, "stderr": 2.985, "cost_per_test": 1.065606, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 70.307, "latency": 454.6, "stderr": 2.674, "cost_per_test": 0.487189, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 70.307, "latency": 2558.912, "stderr": 2.674, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 69.625, "latency": 967.822, "stderr": 2.691, "cost_per_test": 2.63225, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 68.942, "latency": 2910.816, "stderr": 2.708, "cost_per_test": 19.621784, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 68.601, "latency": 1458.474, "stderr": 2.716, "cost_per_test": 1.206641, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 65.87, "latency": 193.59, "stderr": 2.775, "cost_per_test": 0.358329, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 65.87, "latency": 1244.242, "stderr": 2.775, "cost_per_test": 0.32382, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 64.198, "latency": 699.088, "stderr": 3.082, "cost_per_test": 0.77175, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 64.164, "latency": 398.842, "stderr": 2.806, "cost_per_test": 0.035653, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.5": { "accuracy": 61.775, "latency": 662.514, "stderr": 2.844, "cost_per_test": 2.900873, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 59.727, "latency": 315.706, "stderr": 2.87, "cost_per_test": 0.157036, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 59.386, "latency": 536.078, "stderr": 2.874, "cost_per_test": 1.644529, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 59.044, "latency": 869.994, "stderr": 2.878, "cost_per_test": 0.309345, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.3": { "accuracy": 58.703, "latency": 160.62, "stderr": 2.881, "cost_per_test": 0.33705, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 58.703, "latency": 322.086, "stderr": 2.881, "cost_per_test": 0.383023, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 54.949, "latency": 3845.16, "stderr": 2.912, "cost_per_test": 7.559756, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 51.195, "latency": 207.128, "stderr": 2.925, "cost_per_test": 0.279031, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 49.147, "latency": 70.562, "stderr": 2.926, "cost_per_test": 0.150699, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 46.758, "latency": 333.714, "stderr": 2.92, "cost_per_test": 0.362288, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 43.686, "latency": 358.07, "stderr": 2.903, "cost_per_test": 0.615337, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 35.836, "latency": 61.06, "stderr": 2.806, "cost_per_test": 0.085625, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 32.423, "latency": 1165.12, "stderr": 2.739, "cost_per_test": 0.238554, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 27.645, "latency": 2319.982, "stderr": 2.617, "cost_per_test": 0.026486, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "criteria_pass_rate__employment_labor": { "meta/muse_spark_1_3_max": { "accuracy": 89.888, "latency": 812.294, "stderr": 1.849, "cost_per_test": 1.542385, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 89.513, "latency": 1012.946, "stderr": 1.879, "cost_per_test": 2.15024, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 89.513, "latency": 1039.986, "stderr": 1.879, "cost_per_test": 1.705269, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 88.764, "latency": 2165.544, "stderr": 1.936, "cost_per_test": 6.38748, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 88.015, "latency": 2613.692, "stderr": 1.991, "cost_per_test": 10.060231, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 87.64, "latency": 1459.546, "stderr": 2.018, "cost_per_test": 14.366748, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 87.266, "latency": 452.27, "stderr": 2.044, "cost_per_test": 0.589655, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 87.266, "latency": 2386.33, "stderr": 2.044, "cost_per_test": 3.463162, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 86.142, "latency": 2960.05, "stderr": 2.118, "cost_per_test": 5.072256, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 85.768, "latency": 2695.91, "stderr": 2.142, "cost_per_test": 1.709298, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.5": { "accuracy": 85.393, "latency": 565.662, "stderr": 2.165, "cost_per_test": 1.185156, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 85.019, "latency": 3521.574, "stderr": 2.188, "cost_per_test": 28.089397, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 84.644, "latency": 727.958, "stderr": 2.211, "cost_per_test": 3.168674, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 84.27, "latency": 1388.314, "stderr": 2.232, "cost_per_test": 16.809333, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 83.895, "latency": 1712.742, "stderr": 2.254, "cost_per_test": 2.622256, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 83.521, "latency": 511.232, "stderr": 2.275, "cost_per_test": 2.671534, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/ember-1": { "accuracy": 83.521, "latency": 1986.27, "stderr": 2.275, "cost_per_test": 2.240265, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 83.146, "latency": 1018.67, "stderr": 2.295, "cost_per_test": 0.176119, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 83.146, "latency": 3527.114, "stderr": 2.295, "cost_per_test": 17.260921, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 82.772, "latency": 1324.74, "stderr": 2.315, "cost_per_test": 7.024491, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 82.397, "latency": 1375.442, "stderr": 2.335, "cost_per_test": 9.475216, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 82.397, "latency": 2217.096, "stderr": 2.335, "cost_per_test": 3.677382, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 82.397, "latency": 2955.226, "stderr": 2.335, "cost_per_test": 18.043089, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 82.397, "latency": 3234.68, "stderr": 2.335, "cost_per_test": 0.382305, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 82.022, "latency": 449.11, "stderr": 2.354, "cost_per_test": 0.13211, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 82.022, "latency": 1928.642, "stderr": 2.354, "cost_per_test": 7.141814, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 81.273, "latency": 1191.516, "stderr": 2.392, "cost_per_test": 1.421054, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 81.273, "latency": 1978.914, "stderr": 2.392, "cost_per_test": 0.989703, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 81.273, "latency": 2260.078, "stderr": 2.392, "cost_per_test": 2.442027, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 80.899, "latency": 888.802, "stderr": 2.41, "cost_per_test": 0.067578, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 80.899, "latency": 1699.206, "stderr": 2.41, "cost_per_test": 0.881184, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 80.524, "latency": 1242.726, "stderr": 2.428, "cost_per_test": 0.135179, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 80.524, "latency": 1577.85, "stderr": 2.428, "cost_per_test": 0.045526, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 79.401, "latency": 803.834, "stderr": 2.48, "cost_per_test": 3.000761, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 79.026, "latency": 704.448, "stderr": 2.496, "cost_per_test": 2.288046, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 78.713, "latency": 2860.095, "stderr": 2.887, "cost_per_test": 14.514568, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 78.652, "latency": 1114.998, "stderr": 2.512, "cost_per_test": 2.612081, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 77.903, "latency": 1138.112, "stderr": 2.544, "cost_per_test": 0.205481, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 77.903, "latency": 1162.312, "stderr": 2.544, "cost_per_test": 1.29091, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 77.154, "latency": 1238.388, "stderr": 2.574, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 76.779, "latency": 515.328, "stderr": 2.589, "cost_per_test": 0.090025, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 76.779, "latency": 546.488, "stderr": 2.589, "cost_per_test": 0.284762, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 76.779, "latency": 1131.366, "stderr": 2.589, "cost_per_test": 1.795425, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 76.779, "latency": 1382.556, "stderr": 2.589, "cost_per_test": 1.421157, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.5": { "accuracy": 76.404, "latency": 554.404, "stderr": 2.603, "cost_per_test": 2.284177, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 76.404, "latency": 709.762, "stderr": 2.603, "cost_per_test": 2.404487, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 74.906, "latency": 465.318, "stderr": 2.658, "cost_per_test": 0.691985, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-sol": { "accuracy": 74.906, "latency": 865.958, "stderr": 2.658, "cost_per_test": 2.274689, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 73.034, "latency": 1280.342, "stderr": 2.721, "cost_per_test": 0.735629, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 72.285, "latency": 719.954, "stderr": 2.744, "cost_per_test": 1.403464, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.1": { "accuracy": 71.91, "latency": 562.82, "stderr": 2.756, "cost_per_test": 0.552987, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 70.412, "latency": 315.964, "stderr": 2.799, "cost_per_test": 0.041572, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 70.412, "latency": 509.998, "stderr": 2.799, "cost_per_test": 2.207924, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 68.539, "latency": 1373.15, "stderr": 2.847, "cost_per_test": 0.154545, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 67.041, "latency": 1148.912, "stderr": 2.882, "cost_per_test": 1.781902, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 67.041, "latency": 1419.994, "stderr": 2.882, "cost_per_test": 7.485275, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 65.543, "latency": 121.512, "stderr": 2.914, "cost_per_test": 0.340745, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 65.543, "latency": 416.436, "stderr": 2.914, "cost_per_test": 0.033177, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 64.794, "latency": 790.09, "stderr": 2.928, "cost_per_test": 1.501786, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 64.045, "latency": 133.024, "stderr": 2.942, "cost_per_test": 0.03689, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 62.921, "latency": 1354.046, "stderr": 2.962, "cost_per_test": 0.293256, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 61.538, "latency": 1204.09, "stderr": 6.812, "cost_per_test": 0.138471, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 58.052, "latency": 4102.402, "stderr": 3.026, "cost_per_test": 6.850325, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 57.303, "latency": 626.114, "stderr": 3.033, "cost_per_test": 0.879911, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 56.18, "latency": 421.736, "stderr": 3.042, "cost_per_test": 0.169864, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.3": { "accuracy": 55.431, "latency": 121.968, "stderr": 3.048, "cost_per_test": 0.276528, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 54.682, "latency": 467.764, "stderr": 3.052, "cost_per_test": 0.514756, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 53.933, "latency": 327.174, "stderr": 3.056, "cost_per_test": 0.361893, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "poolside/laguna-m.1": { "accuracy": 49.438, "latency": 434.874, "stderr": 3.066, "cost_per_test": 0.585035, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 49.064, "latency": 378.896, "stderr": 3.065, "cost_per_test": 0.810734, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 41.573, "latency": 389.886, "stderr": 3.022, "cost_per_test": 0.121458, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 40.824, "latency": 80.69, "stderr": 3.014, "cost_per_test": 0.179064, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 37.453, "latency": 526.302, "stderr": 2.968, "cost_per_test": 0.921211, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 30.337, "latency": 697.154, "stderr": 2.819, "cost_per_test": 0.018184, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 26.966, "latency": 216.774, "stderr": 2.721, "cost_per_test": 0.239029, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 22.846, "latency": 45.712, "stderr": 2.574, "cost_per_test": 0.053854, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "criteria_pass_rate__energy_natural_resources": { "meta/muse_spark_1_2": { "accuracy": 98.68, "latency": 2031.21, "stderr": 0.657, "cost_per_test": 3.505056, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 97.69, "latency": 907.39, "stderr": 0.864, "cost_per_test": 3.111032, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 97.03, "latency": 1738.252, "stderr": 0.977, "cost_per_test": 17.557582, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 96.7, "latency": 928.796, "stderr": 1.028, "cost_per_test": 6.265695, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 96.7, "latency": 1538.616, "stderr": 1.028, "cost_per_test": 0.224328, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.6": { "accuracy": 96.7, "latency": 2061.922, "stderr": 1.028, "cost_per_test": 4.142205, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 96.7, "latency": 3028.416, "stderr": 1.028, "cost_per_test": 14.055481, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 96.37, "latency": 1031.512, "stderr": 1.076, "cost_per_test": 1.224794, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 96.37, "latency": 1540.416, "stderr": 1.076, "cost_per_test": 0.125628, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 96.37, "latency": 1638.978, "stderr": 1.076, "cost_per_test": 7.731016, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 96.37, "latency": 1654.116, "stderr": 1.076, "cost_per_test": 3.699196, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 96.37, "latency": 1767.97, "stderr": 1.076, "cost_per_test": 1.160357, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 96.37, "latency": 1774.802, "stderr": 1.076, "cost_per_test": 0.288516, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 96.04, "latency": 1808.578, "stderr": 1.122, "cost_per_test": 32.1783, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 95.71, "latency": 447.692, "stderr": 1.166, "cost_per_test": 3.885089, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3": { "accuracy": 95.71, "latency": 1996.308, "stderr": 1.166, "cost_per_test": 5.233643, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 95.71, "latency": 2531.316, "stderr": 1.166, "cost_per_test": 3.273878, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 95.71, "latency": 2822.94, "stderr": 1.166, "cost_per_test": 5.852595, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 95.38, "latency": 1405.156, "stderr": 1.208, "cost_per_test": 9.063448, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "tencent/hy4-preview": { "accuracy": 95.38, "latency": 1891.152, "stderr": 1.208, "cost_per_test": 1.207559, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 95.38, "latency": 3374.346, "stderr": 1.208, "cost_per_test": 4.405884, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 95.05, "latency": 685.118, "stderr": 1.248, "cost_per_test": 3.669584, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 95.05, "latency": 1396.084, "stderr": 1.248, "cost_per_test": 4.475508, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 95.05, "latency": 3747.78, "stderr": 1.248, "cost_per_test": 31.649398, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 94.719, "latency": 2907.838, "stderr": 1.287, "cost_per_test": 4.82694, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 94.719, "latency": 2954.94, "stderr": 1.287, "cost_per_test": 0.562571, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 94.444, "latency": 4775.52, "stderr": 1.712, "cost_per_test": 9.063118, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.7": { "accuracy": 94.389, "latency": 1988.198, "stderr": 1.324, "cost_per_test": 10.743255, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 94.059, "latency": 1183.832, "stderr": 1.36, "cost_per_test": 0.100514, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 94.059, "latency": 1560.884, "stderr": 1.36, "cost_per_test": 14.535287, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 93.729, "latency": 1415.76, "stderr": 1.395, "cost_per_test": 11.82685, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 93.399, "latency": 388.468, "stderr": 1.429, "cost_per_test": 0.983444, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 93.399, "latency": 1700.96, "stderr": 1.429, "cost_per_test": 1.80131, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 93.069, "latency": 398.398, "stderr": 1.461, "cost_per_test": 0.219636, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-sol": { "accuracy": 92.739, "latency": 1169.822, "stderr": 1.493, "cost_per_test": 8.24435, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 92.739, "latency": 1722.806, "stderr": 1.493, "cost_per_test": 17.774066, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 92.409, "latency": 1943.202, "stderr": 1.524, "cost_per_test": 50.660049, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 91.749, "latency": 753.42, "stderr": 1.583, "cost_per_test": 0.613118, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 91.419, "latency": 2085.852, "stderr": 1.612, "cost_per_test": 13.632671, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 90.759, "latency": 1571.69, "stderr": 1.666, "cost_per_test": 4.843052, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 89.769, "latency": 1207.448, "stderr": 1.744, "cost_per_test": 0.670627, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 88.779, "latency": 506.004, "stderr": 1.816, "cost_per_test": 0.05328, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 88.449, "latency": 1484.988, "stderr": 1.839, "cost_per_test": 1.559743, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 88.119, "latency": 1203.204, "stderr": 1.862, "cost_per_test": 1.814778, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 87.459, "latency": 1088.188, "stderr": 1.906, "cost_per_test": 9.258809, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 85.479, "latency": 458.446, "stderr": 2.027, "cost_per_test": 2.006452, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 84.158, "latency": 965.426, "stderr": 2.101, "cost_per_test": 6.052018, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 83.498, "latency": 411.296, "stderr": 2.136, "cost_per_test": 0.138008, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 83.498, "latency": 413.794, "stderr": 2.136, "cost_per_test": 0.042392, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.2": { "accuracy": 79.208, "latency": 853.212, "stderr": 2.335, "cost_per_test": 2.004243, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 77.143, "latency": 161.315, "stderr": 2.688, "cost_per_test": 0.058036, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 74.257, "latency": 368.87, "stderr": 2.516, "cost_per_test": 0.181632, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 73.927, "latency": 360.398, "stderr": 2.526, "cost_per_test": 0.330094, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 71.947, "latency": 7358.538, "stderr": 2.585, "cost_per_test": 54.268239, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 70.627, "latency": 2561.954, "stderr": 2.621, "cost_per_test": 0.226112, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.3": { "accuracy": 69.637, "latency": 204.76, "stderr": 2.646, "cost_per_test": 0.350581, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 67.987, "latency": 815.562, "stderr": 2.685, "cost_per_test": 1.042261, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 64.683, "latency": 3653.828, "stderr": 3.017, "cost_per_test": 26.383042, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 62.222, "latency": 1433.85, "stderr": 3.624, "cost_per_test": 0.330464, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 57.812, "latency": 512.687, "stderr": 3.573, "cost_per_test": 0.30825, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 55.446, "latency": 340.014, "stderr": 2.86, "cost_per_test": 0.495202, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 54.125, "latency": 384.886, "stderr": 2.867, "cost_per_test": 0.461391, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 52.145, "latency": 350.94, "stderr": 2.875, "cost_per_test": 1.916498, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 51.815, "latency": 953.146, "stderr": 2.875, "cost_per_test": 0.646701, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 48.185, "latency": 327.076, "stderr": 2.875, "cost_per_test": 0.37291, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 38.284, "latency": 87.666, "stderr": 2.797, "cost_per_test": 0.205085, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 34.323, "latency": 411.552, "stderr": 2.732, "cost_per_test": 0.399543, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 33.003, "latency": 1976.13, "stderr": 2.706, "cost_per_test": 3.629968, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 31.944, "latency": 1391.76, "stderr": 5.534, "cost_per_test": 0.539658, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 31.353, "latency": 1167.466, "stderr": 2.67, "cost_per_test": 0.032069, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 30.363, "latency": 3233.776, "stderr": 2.646, "cost_per_test": 17.609237, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "inception/mercury-2.5": { "accuracy": 28.053, "latency": 382.348, "stderr": 2.585, "cost_per_test": 0.164729, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 25.311, "latency": 608.292, "stderr": 2.807, "cost_per_test": 0.891986, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 21.122, "latency": 93.152, "stderr": 2.349, "cost_per_test": 0.235869, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 7.755, "latency": 1283.055, "stderr": 1.712, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "thinkingmachines/inkling": { "accuracy": 7.591, "latency": 601.502, "stderr": 1.524, "cost_per_test": 0.939334, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__environmental_esg": { "meta/muse_spark_1_3_max": { "accuracy": 95.402, "latency": 891.213, "stderr": 1.592, "cost_per_test": 1.983442, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 95.402, "latency": 987.19, "stderr": 1.592, "cost_per_test": 1.971553, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 95.341, "latency": 2321.664, "stderr": 1.264, "cost_per_test": 8.278315, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 93.678, "latency": 1476.363, "stderr": 1.85, "cost_per_test": 1.666729, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 92.529, "latency": 496.367, "stderr": 1.999, "cost_per_test": 0.594901, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.5": { "accuracy": 91.398, "latency": 765.464, "stderr": 1.682, "cost_per_test": 2.260337, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 89.606, "latency": 666.612, "stderr": 1.83, "cost_per_test": 3.428704, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 88.889, "latency": 1606.52, "stderr": 1.885, "cost_per_test": 3.219091, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3": { "accuracy": 88.889, "latency": 3334.934, "stderr": 1.885, "cost_per_test": 4.093269, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.6": { "accuracy": 88.889, "latency": 4334.738, "stderr": 1.885, "cost_per_test": 3.914912, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 88.53, "latency": 2090.494, "stderr": 1.911, "cost_per_test": 2.2076, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 88.172, "latency": 1250.098, "stderr": 1.937, "cost_per_test": 0.232534, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 88.172, "latency": 2862.154, "stderr": 1.937, "cost_per_test": 11.199341, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 87.814, "latency": 942.3, "stderr": 1.962, "cost_per_test": 3.664358, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.2": { "accuracy": 87.814, "latency": 1423.762, "stderr": 1.962, "cost_per_test": 1.886054, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 87.814, "latency": 1679.81, "stderr": 1.962, "cost_per_test": 17.039801, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 86.738, "latency": 1687.488, "stderr": 2.034, "cost_per_test": 10.410854, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 86.38, "latency": 1434.172, "stderr": 2.057, "cost_per_test": 17.289629, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 86.38, "latency": 1922.718, "stderr": 2.057, "cost_per_test": 0.97515, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 86.38, "latency": 2749.164, "stderr": 2.057, "cost_per_test": 3.563589, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 85.663, "latency": 1138.234, "stderr": 2.102, "cost_per_test": 3.211685, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 85.663, "latency": 1871.692, "stderr": 2.102, "cost_per_test": 0.897665, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "fireworks/ember-1": { "accuracy": 85.663, "latency": 2493.466, "stderr": 2.102, "cost_per_test": 2.821772, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 85.663, "latency": 9176.072, "stderr": 2.102, "cost_per_test": 54.111114, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 85.305, "latency": 664.552, "stderr": 2.124, "cost_per_test": 3.055555, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 85.305, "latency": 723.794, "stderr": 2.124, "cost_per_test": 0.048721, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 85.305, "latency": 2387.788, "stderr": 2.124, "cost_per_test": 2.678325, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 84.588, "latency": 698.84, "stderr": 2.166, "cost_per_test": 0.167209, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 84.588, "latency": 903.288, "stderr": 2.166, "cost_per_test": 0.086836, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 84.229, "latency": 3980.458, "stderr": 2.186, "cost_per_test": 0.493606, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 83.871, "latency": 412.29, "stderr": 2.206, "cost_per_test": 0.64771, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 83.871, "latency": 2817.198, "stderr": 2.206, "cost_per_test": 8.754557, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 83.513, "latency": 1009.74, "stderr": 2.226, "cost_per_test": 1.92219, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 83.513, "latency": 1732.44, "stderr": 2.226, "cost_per_test": 1.768355, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 83.513, "latency": 1796.954, "stderr": 2.226, "cost_per_test": 0.168238, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 83.154, "latency": 2788.346, "stderr": 2.245, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 83.154, "latency": 4047.096, "stderr": 2.245, "cost_per_test": 18.079139, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 82.437, "latency": 1935.266, "stderr": 2.282, "cost_per_test": 12.224012, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 82.079, "latency": 762.87, "stderr": 2.3, "cost_per_test": 3.081603, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 81.004, "latency": 600.832, "stderr": 2.353, "cost_per_test": 0.102528, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-sol": { "accuracy": 81.004, "latency": 843.6, "stderr": 2.353, "cost_per_test": 2.449822, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 81.004, "latency": 1066.538, "stderr": 2.353, "cost_per_test": 0.207105, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 81.004, "latency": 1902.6, "stderr": 2.353, "cost_per_test": 1.529383, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 80.645, "latency": 1111.074, "stderr": 2.37, "cost_per_test": 7.575723, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 79.928, "latency": 1030.806, "stderr": 2.402, "cost_per_test": 2.99183, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 79.57, "latency": 2481.084, "stderr": 2.418, "cost_per_test": 4.101782, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 79.211, "latency": 510.966, "stderr": 2.434, "cost_per_test": 0.033992, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 79.211, "latency": 1709.262, "stderr": 2.434, "cost_per_test": 0.994698, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 78.853, "latency": 3502.29, "stderr": 2.449, "cost_per_test": 24.257857, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 78.495, "latency": 612.728, "stderr": 2.464, "cost_per_test": 0.620051, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 77.419, "latency": 746.228, "stderr": 2.508, "cost_per_test": 0.310354, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 76.344, "latency": 484.558, "stderr": 2.549, "cost_per_test": 0.058194, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 72.76, "latency": 470.746, "stderr": 2.67, "cost_per_test": 2.036146, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 72.401, "latency": 882.072, "stderr": 2.681, "cost_per_test": 2.235795, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 72.043, "latency": 4134.27, "stderr": 2.692, "cost_per_test": 24.419754, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 71.326, "latency": 1195.644, "stderr": 2.712, "cost_per_test": 3.029146, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 70.968, "latency": 157.04, "stderr": 2.722, "cost_per_test": 0.040915, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 68.817, "latency": 1402.704, "stderr": 2.778, "cost_per_test": 0.30468, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 68.817, "latency": 1534.666, "stderr": 2.778, "cost_per_test": 0.173828, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 67.742, "latency": 304.116, "stderr": 2.804, "cost_per_test": 0.36933, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 67.025, "latency": 421.46, "stderr": 2.82, "cost_per_test": 0.603349, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 62.366, "latency": 1397.79, "stderr": 2.906, "cost_per_test": 1.189505, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "grok/grok-4.3": { "accuracy": 62.007, "latency": 136.838, "stderr": 2.911, "cost_per_test": 0.279814, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 60.932, "latency": 453.494, "stderr": 2.926, "cost_per_test": 0.562732, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 60.932, "latency": 702.338, "stderr": 2.926, "cost_per_test": 1.074487, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 58.781, "latency": 223.294, "stderr": 2.952, "cost_per_test": 0.526047, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 55.502, "latency": 1173.572, "stderr": 3.446, "cost_per_test": 0.395489, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 53.763, "latency": 2805.428, "stderr": 2.99, "cost_per_test": 6.10019, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 50.179, "latency": 375.606, "stderr": 2.999, "cost_per_test": 0.483135, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 44.444, "latency": 313.084, "stderr": 2.98, "cost_per_test": 0.22878, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 43.011, "latency": 86.018, "stderr": 2.969, "cost_per_test": 0.222426, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 40.86, "latency": 677.67, "stderr": 2.948, "cost_per_test": 1.19933, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 38.506, "latency": 1322.42, "stderr": 3.7, "cost_per_test": 0.248012, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 36.918, "latency": 154.234, "stderr": 2.894, "cost_per_test": 0.230865, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 29.391, "latency": 59.852, "stderr": 2.732, "cost_per_test": 0.072209, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 15.054, "latency": 2747.75, "stderr": 2.145, "cost_per_test": 0.02734, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "criteria_pass_rate__funds_asset_management": { "google/gemini-4-argon": { "accuracy": 93.003, "latency": 2412.33, "stderr": 1.379, "cost_per_test": 10.601161, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 92.128, "latency": 1163.572, "stderr": 1.456, "cost_per_test": 0.99674, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 90.962, "latency": 950.384, "stderr": 1.55, "cost_per_test": 2.607789, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 90.962, "latency": 1831.468, "stderr": 1.55, "cost_per_test": 2.411987, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 90.087, "latency": 2292.694, "stderr": 1.616, "cost_per_test": 0.275778, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 89.504, "latency": 894.614, "stderr": 1.657, "cost_per_test": 3.568526, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.2": { "accuracy": 88.338, "latency": 1389.99, "stderr": 1.736, "cost_per_test": 2.133545, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 88.338, "latency": 2127.48, "stderr": 1.736, "cost_per_test": 1.258092, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 87.755, "latency": 2651.038, "stderr": 1.773, "cost_per_test": 5.381658, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.7": { "accuracy": 87.755, "latency": 2684.486, "stderr": 1.773, "cost_per_test": 12.823095, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.6": { "accuracy": 87.755, "latency": 3359.162, "stderr": 1.773, "cost_per_test": 4.043474, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 87.172, "latency": 3112.73, "stderr": 1.808, "cost_per_test": 5.130276, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/ember-1": { "accuracy": 86.589, "latency": 3438.736, "stderr": 1.843, "cost_per_test": 4.870552, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 86.589, "latency": 3845.516, "stderr": 1.843, "cost_per_test": 28.228604, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 86.589, "latency": 4279.908, "stderr": 1.843, "cost_per_test": 3.138564, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 86.297, "latency": 985.742, "stderr": 1.859, "cost_per_test": 0.077692, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.5": { "accuracy": 86.006, "latency": 676.124, "stderr": 1.876, "cost_per_test": 2.138169, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 86.006, "latency": 2280.372, "stderr": 1.876, "cost_per_test": 1.20772, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "kimi/kimi-k3": { "accuracy": 85.714, "latency": 963.842, "stderr": 1.892, "cost_per_test": 6.318696, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 85.131, "latency": 400.146, "stderr": 1.924, "cost_per_test": 2.291932, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 84.84, "latency": 2223.198, "stderr": 1.939, "cost_per_test": 32.834156, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 84.548, "latency": 1654.824, "stderr": 1.954, "cost_per_test": 11.982318, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 84.548, "latency": 1687.568, "stderr": 1.954, "cost_per_test": 12.061978, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 84.548, "latency": 3159.604, "stderr": 1.954, "cost_per_test": 4.299795, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 83.965, "latency": 1160.458, "stderr": 1.984, "cost_per_test": 5.475664, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 83.965, "latency": 1393.648, "stderr": 1.984, "cost_per_test": 0.106311, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-astra": { "accuracy": 83.673, "latency": 1928.168, "stderr": 1.999, "cost_per_test": 35.059835, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 83.382, "latency": 2407.934, "stderr": 2.013, "cost_per_test": 10.393235, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 82.507, "latency": 1152.156, "stderr": 2.054, "cost_per_test": 3.25694, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 82.507, "latency": 1624.184, "stderr": 2.054, "cost_per_test": 1.845597, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 82.216, "latency": 1677.146, "stderr": 2.068, "cost_per_test": 0.219769, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 82.216, "latency": 1785.652, "stderr": 2.068, "cost_per_test": 1.192459, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 81.924, "latency": 2262.452, "stderr": 2.081, "cost_per_test": 4.211796, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 81.341, "latency": 1172.158, "stderr": 2.107, "cost_per_test": 2.077847, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 81.05, "latency": 607.44, "stderr": 2.119, "cost_per_test": 0.113815, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 81.05, "latency": 828.736, "stderr": 2.119, "cost_per_test": 0.190615, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 80.758, "latency": 523.878, "stderr": 2.132, "cost_per_test": 0.955566, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 80.758, "latency": 2408.748, "stderr": 2.132, "cost_per_test": 1.625214, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 80.175, "latency": 1081.948, "stderr": 2.156, "cost_per_test": 4.58965, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.1": { "accuracy": 79.883, "latency": 683.726, "stderr": 2.168, "cost_per_test": 0.718616, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 79.592, "latency": 5249.848, "stderr": 2.179, "cost_per_test": 43.038067, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-sol": { "accuracy": 78.717, "latency": 884.656, "stderr": 2.213, "cost_per_test": 3.027072, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 77.551, "latency": 1206.37, "stderr": 2.256, "cost_per_test": 5.020577, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 77.259, "latency": 404.806, "stderr": 2.267, "cost_per_test": 0.057115, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-6-luna": { "accuracy": 77.259, "latency": 1505.88, "stderr": 2.267, "cost_per_test": 0.351581, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 76.968, "latency": 522.94, "stderr": 2.277, "cost_per_test": 2.30202, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 76.385, "latency": 774.918, "stderr": 2.297, "cost_per_test": 0.499947, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 74.927, "latency": 3107.416, "stderr": 2.344, "cost_per_test": 13.033612, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 74.344, "latency": 1298.726, "stderr": 2.362, "cost_per_test": 3.647174, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 74.052, "latency": 512.676, "stderr": 2.37, "cost_per_test": 0.040011, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 71.72, "latency": 3576.656, "stderr": 2.435, "cost_per_test": 0.771439, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 71.429, "latency": 1059.502, "stderr": 2.443, "cost_per_test": 3.179398, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 71.137, "latency": 974.284, "stderr": 2.45, "cost_per_test": 3.406867, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 70.262, "latency": 1369.844, "stderr": 2.472, "cost_per_test": 1.427877, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 68.513, "latency": 2129.294, "stderr": 2.512, "cost_per_test": 0.202207, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 64.103, "latency": 3329.837, "stderr": 2.909, "cost_per_test": 21.45996, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 63.265, "latency": 3375.11, "stderr": 2.607, "cost_per_test": 6.966914, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 62.682, "latency": 309.586, "stderr": 2.615, "cost_per_test": 0.313043, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 62.099, "latency": 569.218, "stderr": 2.623, "cost_per_test": 2.973017, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 57.434, "latency": 460.218, "stderr": 2.674, "cost_per_test": 0.716192, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "poolside/laguna-m.1": { "accuracy": 55.102, "latency": 644.558, "stderr": 2.69, "cost_per_test": 0.636171, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 54.81, "latency": 1783.184, "stderr": 2.691, "cost_per_test": 0.392274, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 54.227, "latency": 754.854, "stderr": 2.694, "cost_per_test": 1.259469, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 53.936, "latency": 245.102, "stderr": 2.695, "cost_per_test": 0.620247, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 53.353, "latency": 344.172, "stderr": 2.698, "cost_per_test": 0.132025, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 51.698, "latency": 3933.64, "stderr": 3.076, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 51.232, "latency": 267.027, "stderr": 3.517, "cost_per_test": 0.116707, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 47.813, "latency": 213.786, "stderr": 2.701, "cost_per_test": 0.705274, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 46.939, "latency": 246.674, "stderr": 2.699, "cost_per_test": 0.061406, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 45.19, "latency": 325.206, "stderr": 2.691, "cost_per_test": 0.438451, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 36.735, "latency": 549.724, "stderr": 2.607, "cost_per_test": 0.736004, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 35.277, "latency": 322.28, "stderr": 2.584, "cost_per_test": 0.253859, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 32.945, "latency": 1338.436, "stderr": 2.542, "cost_per_test": 0.260312, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 30.321, "latency": 4008.234, "stderr": 2.485, "cost_per_test": 19.376575, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 22.449, "latency": 81.95, "stderr": 2.256, "cost_per_test": 0.142138, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 17.201, "latency": 2922.548, "stderr": 2.041, "cost_per_test": 0.051392, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "criteria_pass_rate__healthcare_life_sciences": { "meta/muse_spark_1_2": { "accuracy": 95.38, "latency": 1815.46, "stderr": 1.208, "cost_per_test": 2.602242, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 94.059, "latency": 1130.752, "stderr": 1.36, "cost_per_test": 2.30716, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 93.729, "latency": 554.702, "stderr": 1.395, "cost_per_test": 3.175621, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 93.729, "latency": 833.11, "stderr": 1.395, "cost_per_test": 3.474289, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 93.729, "latency": 909.876, "stderr": 1.395, "cost_per_test": 1.865298, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "fireworks/ember-1": { "accuracy": 93.399, "latency": 2000.958, "stderr": 1.429, "cost_per_test": 2.838311, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 93.399, "latency": 2623.2, "stderr": 1.429, "cost_per_test": 4.871642, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.6": { "accuracy": 93.069, "latency": 2323.636, "stderr": 1.461, "cost_per_test": 4.837427, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.5": { "accuracy": 92.739, "latency": 761.65, "stderr": 1.493, "cost_per_test": 3.027345, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 92.739, "latency": 1001.374, "stderr": 1.493, "cost_per_test": 0.991405, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 92.739, "latency": 2917.028, "stderr": 1.493, "cost_per_test": 8.843854, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 92.409, "latency": 1607.468, "stderr": 1.524, "cost_per_test": 0.09248, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 92.079, "latency": 2184.576, "stderr": 1.554, "cost_per_test": 0.21495, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 92.079, "latency": 2998.624, "stderr": 1.554, "cost_per_test": 2.223213, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 91.749, "latency": 511.32, "stderr": 1.583, "cost_per_test": 3.002272, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 91.749, "latency": 873.216, "stderr": 1.583, "cost_per_test": 2.706824, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 91.419, "latency": 1343.57, "stderr": 1.612, "cost_per_test": 10.479166, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 91.419, "latency": 1360.682, "stderr": 1.612, "cost_per_test": 2.190988, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 91.419, "latency": 1645.034, "stderr": 1.612, "cost_per_test": 23.076834, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 91.089, "latency": 1043.858, "stderr": 1.639, "cost_per_test": 0.056365, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 90.759, "latency": 1207.572, "stderr": 1.666, "cost_per_test": 0.164353, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.7": { "accuracy": 90.759, "latency": 2753.772, "stderr": 1.666, "cost_per_test": 14.517611, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 90.759, "latency": 3583.414, "stderr": 1.666, "cost_per_test": 24.921853, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 90.429, "latency": 622.19, "stderr": 1.693, "cost_per_test": 0.166133, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 90.429, "latency": 2020.436, "stderr": 1.693, "cost_per_test": 1.267174, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3": { "accuracy": 90.099, "latency": 2196.24, "stderr": 1.719, "cost_per_test": 4.04281, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 90.099, "latency": 2200.132, "stderr": 1.719, "cost_per_test": 1.289815, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 89.769, "latency": 1436.112, "stderr": 1.744, "cost_per_test": 1.4604, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 89.109, "latency": 1954.012, "stderr": 1.793, "cost_per_test": 14.122835, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 88.779, "latency": 1277.572, "stderr": 1.816, "cost_per_test": 3.819729, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 88.449, "latency": 4427.734, "stderr": 1.839, "cost_per_test": 41.981832, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 87.789, "latency": 541.786, "stderr": 1.884, "cost_per_test": 0.121036, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 87.129, "latency": 1131.514, "stderr": 1.927, "cost_per_test": 1.902096, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 87.129, "latency": 3250.102, "stderr": 1.927, "cost_per_test": 0.762902, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 87.129, "latency": 3718.032, "stderr": 1.927, "cost_per_test": 4.342941, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 87.129, "latency": 4075.734, "stderr": 1.927, "cost_per_test": 19.894952, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 86.469, "latency": 1509.924, "stderr": 1.968, "cost_per_test": 20.8409, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 86.139, "latency": 435.376, "stderr": 1.988, "cost_per_test": 0.848233, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 86.139, "latency": 4054.91, "stderr": 1.988, "cost_per_test": 28.864668, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 85.479, "latency": 2340.06, "stderr": 2.027, "cost_per_test": 8.616817, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 84.818, "latency": 2182.822, "stderr": 2.065, "cost_per_test": 1.715622, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 84.488, "latency": 1021.66, "stderr": 2.083, "cost_per_test": 3.327293, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 82.508, "latency": 1594.302, "stderr": 2.186, "cost_per_test": 11.601398, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 81.848, "latency": 812.896, "stderr": 2.218, "cost_per_test": 4.010513, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 80.858, "latency": 997.662, "stderr": 2.264, "cost_per_test": 0.239122, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 80.528, "latency": 444.818, "stderr": 2.279, "cost_per_test": 0.043026, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 80.528, "latency": 575.66, "stderr": 2.279, "cost_per_test": 2.41183, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 79.868, "latency": 802.494, "stderr": 2.307, "cost_per_test": 2.391775, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 79.208, "latency": 2024.916, "stderr": 2.335, "cost_per_test": 0.171673, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 78.878, "latency": 965.994, "stderr": 2.349, "cost_per_test": 3.200117, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 78.548, "latency": 485.53, "stderr": 2.362, "cost_per_test": 0.860793, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 78.218, "latency": 702.102, "stderr": 2.375, "cost_per_test": 0.289523, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 74.917, "latency": 377.358, "stderr": 2.494, "cost_per_test": 0.050679, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 74.257, "latency": 934.792, "stderr": 2.516, "cost_per_test": 2.130891, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 72.607, "latency": 172.172, "stderr": 2.566, "cost_per_test": 0.43244, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 71.053, "latency": 189.678, "stderr": 3.01, "cost_per_test": 0.047604, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 69.637, "latency": 1443.008, "stderr": 2.646, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 69.307, "latency": 496.698, "stderr": 2.654, "cost_per_test": 0.183866, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.1": { "accuracy": 67.987, "latency": 617.856, "stderr": 2.685, "cost_per_test": 0.759264, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 66.534, "latency": 2395.058, "stderr": 2.984, "cost_per_test": 3.397341, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 65.306, "latency": 2007.388, "stderr": 3.047, "cost_per_test": 0.460233, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 63.366, "latency": 1759.028, "stderr": 2.772, "cost_per_test": 0.66545, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 61.716, "latency": 467.984, "stderr": 2.797, "cost_per_test": 0.463686, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 60.396, "latency": 141.404, "stderr": 2.814, "cost_per_test": 0.284177, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 60.396, "latency": 597.642, "stderr": 2.814, "cost_per_test": 0.741325, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 60.066, "latency": 511.986, "stderr": 2.818, "cost_per_test": 0.572801, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 59.076, "latency": 215.342, "stderr": 2.829, "cost_per_test": 0.288454, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 55.446, "latency": 4695.758, "stderr": 2.86, "cost_per_test": 6.264992, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 55.102, "latency": 772.625, "stderr": 3.184, "cost_per_test": 1.351574, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 43.234, "latency": 1477.564, "stderr": 2.851, "cost_per_test": 1.476459, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 42.904, "latency": 386.764, "stderr": 2.848, "cost_per_test": 0.191612, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 42.904, "latency": 1426.678, "stderr": 2.848, "cost_per_test": 0.336544, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "inception/mercury-2.5": { "accuracy": 42.574, "latency": 112.726, "stderr": 2.845, "cost_per_test": 0.210725, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 42.574, "latency": 2823.914, "stderr": 2.845, "cost_per_test": 0.027595, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 34.983, "latency": 691.188, "stderr": 2.744, "cost_per_test": 1.096454, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 30.033, "latency": 46.904, "stderr": 2.638, "cost_per_test": 0.049241, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "criteria_pass_rate__immigration": { "meta/muse_spark_1_2": { "accuracy": 94.118, "latency": 1287.105, "stderr": 1.586, "cost_per_test": 1.326868, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 93.213, "latency": 529.208, "stderr": 1.696, "cost_per_test": 0.483627, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 93.091, "latency": 3374.932, "stderr": 1.532, "cost_per_test": 3.07805, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 92.76, "latency": 846.297, "stderr": 1.747, "cost_per_test": 1.891702, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 92.364, "latency": 2775.568, "stderr": 1.604, "cost_per_test": 9.272635, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "tencent/hy4-preview": { "accuracy": 92.0, "latency": 1834.206, "stderr": 1.639, "cost_per_test": 0.79121, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3": { "accuracy": 91.273, "latency": 2299.654, "stderr": 1.705, "cost_per_test": 3.068155, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/ember-1": { "accuracy": 91.273, "latency": 2833.772, "stderr": 1.705, "cost_per_test": 2.00223, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 90.909, "latency": 592.33, "stderr": 1.737, "cost_per_test": 2.608087, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 90.909, "latency": 2574.232, "stderr": 1.737, "cost_per_test": 9.207192, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 90.182, "latency": 757.904, "stderr": 1.798, "cost_per_test": 2.686527, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 90.182, "latency": 1489.498, "stderr": 1.798, "cost_per_test": 14.198276, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 90.182, "latency": 2995.124, "stderr": 1.798, "cost_per_test": 19.864027, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 90.045, "latency": 859.238, "stderr": 2.019, "cost_per_test": 1.580757, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.5": { "accuracy": 89.091, "latency": 539.64, "stderr": 1.883, "cost_per_test": 1.422373, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 88.364, "latency": 770.674, "stderr": 1.937, "cost_per_test": 2.857841, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 88.364, "latency": 3260.778, "stderr": 1.937, "cost_per_test": 13.84088, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 88.0, "latency": 985.112, "stderr": 1.963, "cost_per_test": 0.069274, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.2": { "accuracy": 88.0, "latency": 1326.436, "stderr": 1.963, "cost_per_test": 1.8212, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 88.0, "latency": 2101.604, "stderr": 1.963, "cost_per_test": 2.214603, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 87.636, "latency": 348.37, "stderr": 1.989, "cost_per_test": 2.310037, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "stepfun/step-5-preview": { "accuracy": 87.636, "latency": 1558.018, "stderr": 1.989, "cost_per_test": 0.755618, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 87.636, "latency": 3189.722, "stderr": 1.989, "cost_per_test": 25.924114, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 87.442, "latency": 2130.83, "stderr": 2.265, "cost_per_test": 2.704387, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 87.273, "latency": 2903.422, "stderr": 2.013, "cost_per_test": 18.52264, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 87.273, "latency": 2972.516, "stderr": 2.013, "cost_per_test": 0.178109, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 86.909, "latency": 1971.066, "stderr": 2.038, "cost_per_test": 8.081965, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 86.545, "latency": 1046.02, "stderr": 2.061, "cost_per_test": 2.059771, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 86.545, "latency": 1687.584, "stderr": 2.061, "cost_per_test": 1.581573, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 85.818, "latency": 589.338, "stderr": 2.108, "cost_per_test": 0.152164, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 85.818, "latency": 1347.378, "stderr": 2.108, "cost_per_test": 7.846506, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 85.455, "latency": 537.584, "stderr": 2.13, "cost_per_test": 0.513525, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 85.091, "latency": 666.162, "stderr": 2.152, "cost_per_test": 0.041542, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-sol": { "accuracy": 84.727, "latency": 616.432, "stderr": 2.173, "cost_per_test": 1.548993, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 84.727, "latency": 1172.736, "stderr": 2.173, "cost_per_test": 11.161235, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 84.364, "latency": 333.842, "stderr": 2.194, "cost_per_test": 0.519471, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "thinkingmachines/inkling": { "accuracy": 84.364, "latency": 707.206, "stderr": 2.194, "cost_per_test": 0.851971, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 84.364, "latency": 1017.402, "stderr": 2.194, "cost_per_test": 5.823174, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 84.364, "latency": 1119.492, "stderr": 2.194, "cost_per_test": 0.128623, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 84.364, "latency": 3031.804, "stderr": 2.194, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 83.636, "latency": 2453.73, "stderr": 2.235, "cost_per_test": 0.399209, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 82.909, "latency": 634.518, "stderr": 2.274, "cost_per_test": 0.083903, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 82.545, "latency": 1456.41, "stderr": 2.293, "cost_per_test": 8.05746, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 82.182, "latency": 1062.394, "stderr": 2.312, "cost_per_test": 0.987413, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-6-luna": { "accuracy": 81.818, "latency": 630.912, "stderr": 2.33, "cost_per_test": 0.110658, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 81.818, "latency": 682.574, "stderr": 2.33, "cost_per_test": 2.985935, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 81.818, "latency": 844.856, "stderr": 2.33, "cost_per_test": 1.515755, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 80.0, "latency": 2243.912, "stderr": 2.416, "cost_per_test": 1.390074, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 79.273, "latency": 328.71, "stderr": 2.449, "cost_per_test": 0.042242, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 78.545, "latency": 575.12, "stderr": 2.48, "cost_per_test": 0.227351, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 78.182, "latency": 1322.102, "stderr": 2.495, "cost_per_test": 0.650266, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 77.818, "latency": 783.178, "stderr": 2.51, "cost_per_test": 1.301215, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 77.455, "latency": 2152.852, "stderr": 2.525, "cost_per_test": 6.833464, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 76.727, "latency": 419.566, "stderr": 2.553, "cost_per_test": 1.560451, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 76.727, "latency": 1555.058, "stderr": 2.553, "cost_per_test": 0.160872, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 75.636, "latency": 456.404, "stderr": 2.593, "cost_per_test": 0.028647, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 75.273, "latency": 1442.084, "stderr": 2.606, "cost_per_test": 0.342586, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 74.909, "latency": 694.87, "stderr": 2.619, "cost_per_test": 1.962883, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 74.182, "latency": 126.21, "stderr": 2.644, "cost_per_test": 0.037666, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 73.818, "latency": 444.094, "stderr": 2.656, "cost_per_test": 0.391182, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 72.364, "latency": 771.262, "stderr": 2.702, "cost_per_test": 2.396939, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 71.273, "latency": 1803.852, "stderr": 2.734, "cost_per_test": 2.10625, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 70.545, "latency": 431.31, "stderr": 2.754, "cost_per_test": 0.284081, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.3": { "accuracy": 66.182, "latency": 148.65, "stderr": 2.858, "cost_per_test": 0.35425, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 62.909, "latency": 637.036, "stderr": 2.918, "cost_per_test": 0.459943, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 62.545, "latency": 309.228, "stderr": 2.924, "cost_per_test": 0.127284, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 57.091, "latency": 513.374, "stderr": 2.99, "cost_per_test": 0.762367, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 54.909, "latency": 298.584, "stderr": 3.006, "cost_per_test": 0.74639, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 51.636, "latency": 341.998, "stderr": 3.019, "cost_per_test": 0.36647, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 45.818, "latency": 240.314, "stderr": 3.01, "cost_per_test": 0.846698, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 43.273, "latency": 571.14, "stderr": 2.993, "cost_per_test": 0.151627, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 36.727, "latency": 77.046, "stderr": 2.912, "cost_per_test": 0.149835, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 35.088, "latency": 1798.635, "stderr": 4.49, "cost_per_test": 0.195853, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 26.182, "latency": 55.438, "stderr": 2.656, "cost_per_test": 0.055276, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 21.818, "latency": 121.822, "stderr": 2.495, "cost_per_test": 0.154995, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 18.545, "latency": 232.172, "stderr": 2.348, "cost_per_test": 0.012559, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "criteria_pass_rate__insurance": { "meta/muse_spark_1_3_max": { "accuracy": 96.337, "latency": 845.222, "stderr": 1.139, "cost_per_test": 1.773661, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 94.872, "latency": 3828.232, "stderr": 1.337, "cost_per_test": 3.443111, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 94.139, "latency": 1085.91, "stderr": 1.424, "cost_per_test": 1.256553, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 93.407, "latency": 1417.02, "stderr": 1.505, "cost_per_test": 15.611186, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 93.04, "latency": 1949.21, "stderr": 1.543, "cost_per_test": 6.035978, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 92.308, "latency": 967.18, "stderr": 1.616, "cost_per_test": 2.204869, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 92.308, "latency": 2443.796, "stderr": 1.616, "cost_per_test": 10.158566, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 91.941, "latency": 413.094, "stderr": 1.65, "cost_per_test": 2.436289, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 91.575, "latency": 863.304, "stderr": 1.684, "cost_per_test": 0.168232, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 91.575, "latency": 3746.53, "stderr": 1.684, "cost_per_test": 22.385777, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 91.209, "latency": 1504.748, "stderr": 1.717, "cost_per_test": 0.879072, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "stepfun/step-5-preview": { "accuracy": 91.209, "latency": 1827.906, "stderr": 1.717, "cost_per_test": 1.08578, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "fireworks/ember-1": { "accuracy": 91.209, "latency": 2212.132, "stderr": 1.717, "cost_per_test": 2.324792, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 90.476, "latency": 529.27, "stderr": 1.78, "cost_per_test": 2.685643, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 90.476, "latency": 1077.368, "stderr": 1.78, "cost_per_test": 3.594349, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 89.862, "latency": 2860.728, "stderr": 2.054, "cost_per_test": 16.375474, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 89.744, "latency": 3460.936, "stderr": 1.84, "cost_per_test": 15.744184, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 89.377, "latency": 661.584, "stderr": 1.868, "cost_per_test": 2.129824, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 89.377, "latency": 721.662, "stderr": 1.868, "cost_per_test": 0.060043, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 89.377, "latency": 5962.82, "stderr": 1.868, "cost_per_test": 32.108773, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 88.645, "latency": 802.086, "stderr": 1.924, "cost_per_test": 2.401502, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 88.645, "latency": 986.986, "stderr": 1.924, "cost_per_test": 0.11707, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 88.645, "latency": 1289.65, "stderr": 1.924, "cost_per_test": 15.655481, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 88.645, "latency": 1502.57, "stderr": 1.924, "cost_per_test": 1.707085, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 88.645, "latency": 2233.016, "stderr": 1.924, "cost_per_test": 3.64804, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 88.278, "latency": 3236.886, "stderr": 1.95, "cost_per_test": 0.493207, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 87.931, "latency": 491.63, "stderr": 3.038, "cost_per_test": 0.58862, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 87.912, "latency": 1881.516, "stderr": 1.977, "cost_per_test": 2.905753, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 87.546, "latency": 607.276, "stderr": 2.002, "cost_per_test": 0.042314, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 87.546, "latency": 848.144, "stderr": 2.002, "cost_per_test": 0.151326, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 87.179, "latency": 1205.068, "stderr": 2.027, "cost_per_test": 1.05308, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 87.179, "latency": 1278.736, "stderr": 2.027, "cost_per_test": 9.836148, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.2": { "accuracy": 87.179, "latency": 1287.198, "stderr": 2.027, "cost_per_test": 1.917632, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 87.179, "latency": 1460.754, "stderr": 2.027, "cost_per_test": 9.118767, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 86.813, "latency": 2865.77, "stderr": 2.052, "cost_per_test": 7.165675, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 86.447, "latency": 1209.736, "stderr": 2.075, "cost_per_test": 1.949242, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 86.081, "latency": 2425.114, "stderr": 2.099, "cost_per_test": 2.559806, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 84.615, "latency": 999.666, "stderr": 2.188, "cost_per_test": 2.759668, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 84.507, "latency": 2146.625, "stderr": 2.485, "cost_per_test": 2.868148, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 84.249, "latency": 1239.256, "stderr": 2.209, "cost_per_test": 8.5933, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 82.784, "latency": 631.634, "stderr": 2.289, "cost_per_test": 2.417604, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 82.784, "latency": 824.586, "stderr": 2.289, "cost_per_test": 0.16401, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 82.784, "latency": 3435.596, "stderr": 2.289, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-6-sol": { "accuracy": 82.418, "latency": 762.124, "stderr": 2.308, "cost_per_test": 2.10689, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 82.051, "latency": 2064.252, "stderr": 2.327, "cost_per_test": 1.401873, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 80.952, "latency": 680.89, "stderr": 2.381, "cost_per_test": 0.307965, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 80.586, "latency": 405.378, "stderr": 2.398, "cost_per_test": 0.628366, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.1": { "accuracy": 80.586, "latency": 526.822, "stderr": 2.398, "cost_per_test": 0.601521, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 80.586, "latency": 1140.7, "stderr": 2.398, "cost_per_test": 1.138776, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 80.22, "latency": 1350.64, "stderr": 2.415, "cost_per_test": 0.819488, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 78.755, "latency": 899.01, "stderr": 2.48, "cost_per_test": 2.265639, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 75.458, "latency": 509.878, "stderr": 2.609, "cost_per_test": 0.089394, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 75.092, "latency": 467.072, "stderr": 2.622, "cost_per_test": 0.033672, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 74.725, "latency": 757.324, "stderr": 2.635, "cost_per_test": 1.374128, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 73.993, "latency": 359.782, "stderr": 2.66, "cost_per_test": 0.050634, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 73.26, "latency": 708.134, "stderr": 2.684, "cost_per_test": 1.663014, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 68.132, "latency": 1590.182, "stderr": 2.825, "cost_per_test": 0.269577, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 67.033, "latency": 358.856, "stderr": 2.85, "cost_per_test": 0.368749, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 64.469, "latency": 132.178, "stderr": 2.902, "cost_per_test": 0.044849, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 63.736, "latency": 401.878, "stderr": 2.915, "cost_per_test": 1.769687, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 61.538, "latency": 1985.446, "stderr": 2.95, "cost_per_test": 0.187751, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 58.242, "latency": 412.404, "stderr": 2.99, "cost_per_test": 0.441831, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "poolside/laguna-m.1": { "accuracy": 58.242, "latency": 473.472, "stderr": 2.99, "cost_per_test": 0.555887, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 57.509, "latency": 353.55, "stderr": 2.997, "cost_per_test": 0.247595, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 57.509, "latency": 879.704, "stderr": 2.997, "cost_per_test": 0.471893, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 56.777, "latency": 257.758, "stderr": 3.004, "cost_per_test": 0.666001, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 56.777, "latency": 555.876, "stderr": 3.004, "cost_per_test": 0.801823, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 54.579, "latency": 2551.164, "stderr": 3.019, "cost_per_test": 4.703644, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 49.451, "latency": 611.662, "stderr": 3.032, "cost_per_test": 0.866939, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.3": { "accuracy": 47.985, "latency": 145.796, "stderr": 3.029, "cost_per_test": 0.345132, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 46.154, "latency": 137.038, "stderr": 3.023, "cost_per_test": 0.213394, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 44.843, "latency": 1296.19, "stderr": 3.338, "cost_per_test": 0.286257, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 40.293, "latency": 283.512, "stderr": 2.974, "cost_per_test": 0.376663, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "inception/mercury-2.5": { "accuracy": 39.194, "latency": 103.32, "stderr": 2.96, "cost_per_test": 0.182741, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 23.81, "latency": 73.878, "stderr": 2.583, "cost_per_test": 0.087874, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 19.78, "latency": 1719.334, "stderr": 2.415, "cost_per_test": 0.008422, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "criteria_pass_rate__intellectual_property": { "google/gemini-4-argon": { "accuracy": 96.466, "latency": 2805.298, "stderr": 1.099, "cost_per_test": 10.686247, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 95.76, "latency": 1515.528, "stderr": 1.2, "cost_per_test": 2.100924, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 95.406, "latency": 561.322, "stderr": 1.247, "cost_per_test": 0.701418, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.5": { "accuracy": 95.053, "latency": 474.614, "stderr": 1.291, "cost_per_test": 1.436571, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 95.053, "latency": 808.438, "stderr": 1.291, "cost_per_test": 2.154605, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 95.053, "latency": 960.974, "stderr": 1.291, "cost_per_test": 0.088739, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 95.053, "latency": 1036.242, "stderr": 1.291, "cost_per_test": 0.205832, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 94.7, "latency": 1116.818, "stderr": 1.334, "cost_per_test": 2.754711, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 94.7, "latency": 2073.084, "stderr": 1.334, "cost_per_test": 3.723686, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 94.7, "latency": 2640.56, "stderr": 1.334, "cost_per_test": 5.422039, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 93.993, "latency": 610.822, "stderr": 1.415, "cost_per_test": 4.042103, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 93.993, "latency": 9064.15, "stderr": 1.415, "cost_per_test": 48.490754, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 93.64, "latency": 1316.854, "stderr": 1.453, "cost_per_test": 0.743241, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "zai/glm-5.3": { "accuracy": 93.64, "latency": 1969.154, "stderr": 1.453, "cost_per_test": 3.617042, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "fireworks/ember-1": { "accuracy": 93.64, "latency": 2910.156, "stderr": 1.453, "cost_per_test": 3.950111, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 93.286, "latency": 1874.49, "stderr": 1.49, "cost_per_test": 1.123387, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 92.933, "latency": 265.502, "stderr": 1.526, "cost_per_test": 1.928307, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 92.58, "latency": 1487.716, "stderr": 1.561, "cost_per_test": 0.171603, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 92.58, "latency": 1536.328, "stderr": 1.561, "cost_per_test": 19.843555, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 92.58, "latency": 2660.726, "stderr": 1.561, "cost_per_test": 1.882569, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 92.226, "latency": 1397.42, "stderr": 1.594, "cost_per_test": 9.861514, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 92.226, "latency": 1399.04, "stderr": 1.594, "cost_per_test": 20.104733, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 92.226, "latency": 2027.16, "stderr": 1.594, "cost_per_test": 3.241614, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 91.873, "latency": 1008.958, "stderr": 1.627, "cost_per_test": 4.951983, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 91.873, "latency": 1121.5, "stderr": 1.627, "cost_per_test": 0.055806, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 91.845, "latency": 1483.872, "stderr": 1.797, "cost_per_test": 1.757206, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 91.519, "latency": 564.488, "stderr": 1.659, "cost_per_test": 0.166991, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 91.519, "latency": 3194.55, "stderr": 1.659, "cost_per_test": 23.467391, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 91.519, "latency": 3528.822, "stderr": 1.659, "cost_per_test": 17.055475, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 91.166, "latency": 636.93, "stderr": 1.69, "cost_per_test": 0.303093, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 91.166, "latency": 2665.544, "stderr": 1.69, "cost_per_test": 13.072164, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 90.813, "latency": 547.544, "stderr": 1.72, "cost_per_test": 0.109026, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 90.459, "latency": 1246.85, "stderr": 1.749, "cost_per_test": 1.349078, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 90.459, "latency": 1400.054, "stderr": 1.749, "cost_per_test": 0.808637, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 90.459, "latency": 1413.284, "stderr": 1.749, "cost_per_test": 9.530409, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 90.106, "latency": 958.576, "stderr": 1.778, "cost_per_test": 2.381656, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 89.753, "latency": 840.968, "stderr": 1.806, "cost_per_test": 1.83107, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 89.753, "latency": 1954.208, "stderr": 1.806, "cost_per_test": 3.001191, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 89.046, "latency": 924.138, "stderr": 1.86, "cost_per_test": 2.678832, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 88.693, "latency": 2120.66, "stderr": 1.886, "cost_per_test": 3.637047, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 88.693, "latency": 2600.29, "stderr": 1.886, "cost_per_test": 0.569544, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 88.693, "latency": 3064.784, "stderr": 1.886, "cost_per_test": 11.56724, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 88.339, "latency": 380.938, "stderr": 1.911, "cost_per_test": 0.704723, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 88.339, "latency": 432.43, "stderr": 1.911, "cost_per_test": 1.804212, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 88.339, "latency": 939.262, "stderr": 1.911, "cost_per_test": 0.235396, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 87.444, "latency": 3308.745, "stderr": 2.224, "cost_per_test": 20.701364, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 86.572, "latency": 628.006, "stderr": 2.03, "cost_per_test": 3.636776, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 86.219, "latency": 362.93, "stderr": 2.053, "cost_per_test": 0.060908, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "zai/glm-5.1": { "accuracy": 86.219, "latency": 525.112, "stderr": 2.053, "cost_per_test": 0.593024, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 85.866, "latency": 821.328, "stderr": 2.075, "cost_per_test": 3.295337, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 85.512, "latency": 1126.242, "stderr": 2.096, "cost_per_test": 8.439215, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 84.753, "latency": 1406.772, "stderr": 2.413, "cost_per_test": 1.137063, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 84.452, "latency": 395.842, "stderr": 2.158, "cost_per_test": 0.032956, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 84.099, "latency": 2055.65, "stderr": 2.178, "cost_per_test": 0.218964, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 83.039, "latency": 1248.01, "stderr": 2.235, "cost_per_test": 0.363599, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 82.96, "latency": 836.683, "stderr": 2.523, "cost_per_test": 1.506441, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 82.332, "latency": 903.21, "stderr": 2.271, "cost_per_test": 3.326913, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 81.272, "latency": 126.826, "stderr": 2.323, "cost_per_test": 0.042407, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 79.821, "latency": 1112.945, "stderr": 2.694, "cost_per_test": 0.39665, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.2": { "accuracy": 76.325, "latency": 1072.964, "stderr": 2.531, "cost_per_test": 1.749857, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 75.972, "latency": 129.784, "stderr": 2.544, "cost_per_test": 0.400486, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 75.152, "latency": 634.237, "stderr": 3.374, "cost_per_test": 0.915146, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.3": { "accuracy": 73.852, "latency": 136.232, "stderr": 2.617, "cost_per_test": 0.330997, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 69.258, "latency": 3975.454, "stderr": 2.748, "cost_per_test": 5.12045, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 67.491, "latency": 410.16, "stderr": 2.789, "cost_per_test": 0.177096, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 65.724, "latency": 520.944, "stderr": 2.826, "cost_per_test": 0.715833, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-m.1": { "accuracy": 65.371, "latency": 546.476, "stderr": 2.833, "cost_per_test": 0.703517, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 63.604, "latency": 329.608, "stderr": 2.865, "cost_per_test": 0.359272, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 60.071, "latency": 1545.29, "stderr": 2.916, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 54.77, "latency": 279.476, "stderr": 2.964, "cost_per_test": 1.003874, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 50.53, "latency": 91.85, "stderr": 2.977, "cost_per_test": 0.212309, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 44.876, "latency": 1126.738, "stderr": 2.962, "cost_per_test": 0.271727, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 42.403, "latency": 781.45, "stderr": 2.943, "cost_per_test": 0.026849, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 41.696, "latency": 179.362, "stderr": 2.936, "cost_per_test": 0.27552, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 34.629, "latency": 65.61, "stderr": 2.833, "cost_per_test": 0.092629, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 30.389, "latency": 376.326, "stderr": 2.739, "cost_per_test": 0.402281, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null } }, "criteria_pass_rate__international_trade_sanctions": { "meta/muse_spark_1_3": { "accuracy": 94.554, "latency": 1489.08, "stderr": 1.601, "cost_per_test": 1.89638, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 94.079, "latency": 819.89, "stderr": 1.921, "cost_per_test": 1.928195, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 94.048, "latency": 3028.722, "stderr": 1.493, "cost_per_test": 28.213056, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-4-argon": { "accuracy": 93.254, "latency": 2018.01, "stderr": 1.583, "cost_per_test": 7.007018, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 93.254, "latency": 2372.88, "stderr": 1.583, "cost_per_test": 3.054658, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 93.069, "latency": 2085.605, "stderr": 1.791, "cost_per_test": 2.123477, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 92.46, "latency": 1640.224, "stderr": 1.667, "cost_per_test": 15.317586, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 92.063, "latency": 646.68, "stderr": 1.706, "cost_per_test": 2.683796, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 91.27, "latency": 1563.178, "stderr": 1.782, "cost_per_test": 0.192448, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "stepfun/step-5-preview": { "accuracy": 90.873, "latency": 1780.01, "stderr": 1.818, "cost_per_test": 0.89711, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 90.079, "latency": 788.418, "stderr": 1.887, "cost_per_test": 2.340083, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 90.0, "latency": 372.4, "stderr": 4.286, "cost_per_test": 0.355364, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.5": { "accuracy": 89.683, "latency": 534.972, "stderr": 1.92, "cost_per_test": 1.312972, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 89.683, "latency": 955.068, "stderr": 1.92, "cost_per_test": 0.06912, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 89.683, "latency": 2464.414, "stderr": 1.92, "cost_per_test": 4.173567, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.7": { "accuracy": 89.286, "latency": 2737.024, "stderr": 1.952, "cost_per_test": 10.904646, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 89.109, "latency": 643.488, "stderr": 2.197, "cost_per_test": 0.03909, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "fireworks/ember-1": { "accuracy": 88.889, "latency": 1853.572, "stderr": 1.984, "cost_per_test": 2.221756, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 88.492, "latency": 1246.816, "stderr": 2.014, "cost_per_test": 0.1231, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 88.492, "latency": 1639.072, "stderr": 2.014, "cost_per_test": 0.787466, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 88.492, "latency": 3708.434, "stderr": 2.014, "cost_per_test": 16.269947, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 88.095, "latency": 859.204, "stderr": 2.044, "cost_per_test": 3.921448, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 88.095, "latency": 1357.182, "stderr": 2.044, "cost_per_test": 7.899077, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 87.698, "latency": 1995.074, "stderr": 2.073, "cost_per_test": 1.406803, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 87.698, "latency": 2972.586, "stderr": 2.073, "cost_per_test": 20.461751, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 87.698, "latency": 3099.466, "stderr": 2.073, "cost_per_test": 20.677557, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 87.302, "latency": 2305.496, "stderr": 2.102, "cost_per_test": 3.46041, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 86.905, "latency": 555.678, "stderr": 2.129, "cost_per_test": 0.163871, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 86.905, "latency": 1239.718, "stderr": 2.129, "cost_per_test": 13.625923, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 86.508, "latency": 1095.19, "stderr": 2.156, "cost_per_test": 0.888266, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling": { "accuracy": 86.508, "latency": 1108.946, "stderr": 2.156, "cost_per_test": 1.18382, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 85.714, "latency": 1470.566, "stderr": 2.209, "cost_per_test": 7.692416, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 85.317, "latency": 2010.348, "stderr": 2.234, "cost_per_test": 7.464714, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 85.317, "latency": 2161.244, "stderr": 2.234, "cost_per_test": 0.466223, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 85.317, "latency": 2309.216, "stderr": 2.234, "cost_per_test": 2.114472, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 84.921, "latency": 857.59, "stderr": 2.259, "cost_per_test": 1.591969, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 84.921, "latency": 912.208, "stderr": 2.259, "cost_per_test": 2.317735, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 84.524, "latency": 2095.02, "stderr": 2.283, "cost_per_test": 3.310241, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 83.73, "latency": 566.308, "stderr": 2.33, "cost_per_test": 0.257491, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 83.73, "latency": 1627.856, "stderr": 2.33, "cost_per_test": 0.83481, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 83.333, "latency": 676.884, "stderr": 2.352, "cost_per_test": 0.129464, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 82.937, "latency": 320.398, "stderr": 2.374, "cost_per_test": 0.035706, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 82.937, "latency": 461.638, "stderr": 2.374, "cost_per_test": 0.642446, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 82.54, "latency": 1963.404, "stderr": 2.396, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 81.746, "latency": 631.42, "stderr": 2.438, "cost_per_test": 0.093679, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-sol": { "accuracy": 81.746, "latency": 751.328, "stderr": 2.438, "cost_per_test": 2.96028, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 81.746, "latency": 1055.988, "stderr": 2.438, "cost_per_test": 2.391323, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 80.952, "latency": 636.396, "stderr": 2.479, "cost_per_test": 2.359423, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 80.556, "latency": 1023.826, "stderr": 2.498, "cost_per_test": 6.235021, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 80.159, "latency": 632.276, "stderr": 2.517, "cost_per_test": 0.653116, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 79.762, "latency": 397.638, "stderr": 2.536, "cost_per_test": 0.026269, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 79.365, "latency": 2245.596, "stderr": 2.554, "cost_per_test": 1.091719, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 78.968, "latency": 521.298, "stderr": 2.572, "cost_per_test": 0.564526, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 78.571, "latency": 414.772, "stderr": 2.59, "cost_per_test": 1.880602, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 78.571, "latency": 1052.306, "stderr": 2.59, "cost_per_test": 1.377178, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 77.778, "latency": 1375.622, "stderr": 2.624, "cost_per_test": 0.294691, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 77.381, "latency": 2228.33, "stderr": 2.641, "cost_per_test": 0.192339, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 76.984, "latency": 728.318, "stderr": 2.657, "cost_per_test": 1.85423, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 74.603, "latency": 750.932, "stderr": 2.747, "cost_per_test": 1.851192, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 73.81, "latency": 271.008, "stderr": 2.775, "cost_per_test": 0.333493, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 72.222, "latency": 151.836, "stderr": 2.827, "cost_per_test": 0.040613, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 71.032, "latency": 438.756, "stderr": 2.863, "cost_per_test": 0.359741, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.2": { "accuracy": 71.032, "latency": 1147.412, "stderr": 2.863, "cost_per_test": 1.320403, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 70.792, "latency": 762.608, "stderr": 3.207, "cost_per_test": 1.028322, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 68.254, "latency": 673.618, "stderr": 2.938, "cost_per_test": 0.52976, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4.3": { "accuracy": 67.063, "latency": 116.996, "stderr": 2.967, "cost_per_test": 0.296944, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 64.356, "latency": 432.125, "stderr": 3.378, "cost_per_test": 0.641757, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 59.524, "latency": 86.68, "stderr": 3.098, "cost_per_test": 0.180981, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 55.556, "latency": 392.874, "stderr": 3.136, "cost_per_test": 0.428518, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 55.556, "latency": 1939.546, "stderr": 3.136, "cost_per_test": 0.023491, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 51.984, "latency": 551.724, "stderr": 3.153, "cost_per_test": 0.590442, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 50.397, "latency": 284.56, "stderr": 3.156, "cost_per_test": 0.166402, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 45.635, "latency": 2783.904, "stderr": 3.144, "cost_per_test": 3.380109, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 40.873, "latency": 292.1, "stderr": 3.103, "cost_per_test": 0.228541, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 32.143, "latency": 47.572, "stderr": 2.948, "cost_per_test": 0.051418, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 0.0, "latency": null, "stderr": 0.0, "cost_per_test": null, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null } }, "criteria_pass_rate__litigation_dispute_resolution": { "meta/muse_spark_1_2": { "accuracy": 95.131, "latency": 1195.492, "stderr": 1.32, "cost_per_test": 2.134092, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 94.382, "latency": 1032.806, "stderr": 1.412, "cost_per_test": 3.090873, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 94.007, "latency": 2587.706, "stderr": 1.455, "cost_per_test": 4.201399, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 93.633, "latency": 748.106, "stderr": 1.497, "cost_per_test": 0.879933, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 93.633, "latency": 875.784, "stderr": 1.497, "cost_per_test": 2.376132, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "fireworks/ember-1": { "accuracy": 92.884, "latency": 2616.116, "stderr": 1.576, "cost_per_test": 3.105393, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 92.884, "latency": 2932.168, "stderr": 1.576, "cost_per_test": 10.28651, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 91.76, "latency": 560.644, "stderr": 1.686, "cost_per_test": 3.560829, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.7": { "accuracy": 91.386, "latency": 2534.012, "stderr": 1.72, "cost_per_test": 10.15094, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.5": { "accuracy": 90.637, "latency": 666.44, "stderr": 1.786, "cost_per_test": 2.62856, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 90.637, "latency": 1720.822, "stderr": 1.786, "cost_per_test": 0.94901, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 90.262, "latency": 1043.246, "stderr": 1.818, "cost_per_test": 0.089255, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 90.262, "latency": 3753.636, "stderr": 1.818, "cost_per_test": 2.324679, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 89.888, "latency": 1464.802, "stderr": 1.849, "cost_per_test": 10.4087, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 89.888, "latency": 2372.292, "stderr": 1.849, "cost_per_test": 4.594796, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3": { "accuracy": 89.888, "latency": 2703.37, "stderr": 1.849, "cost_per_test": 4.259886, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 89.513, "latency": 852.802, "stderr": 1.879, "cost_per_test": 3.706073, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 89.513, "latency": 1029.628, "stderr": 1.879, "cost_per_test": 2.465192, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 89.513, "latency": 1622.368, "stderr": 1.879, "cost_per_test": 0.914139, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 89.513, "latency": 2828.388, "stderr": 1.879, "cost_per_test": 4.311466, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 89.139, "latency": 1467.622, "stderr": 1.908, "cost_per_test": 16.209915, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 88.764, "latency": 403.072, "stderr": 1.936, "cost_per_test": 2.593296, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 88.764, "latency": 1402.92, "stderr": 1.936, "cost_per_test": 1.540985, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 88.764, "latency": 2281.146, "stderr": 1.936, "cost_per_test": 0.217599, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 88.39, "latency": 642.88, "stderr": 1.964, "cost_per_test": 0.187536, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 87.64, "latency": 1542.606, "stderr": 2.018, "cost_per_test": 33.042054, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 86.891, "latency": 636.384, "stderr": 2.069, "cost_per_test": 0.488931, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 86.891, "latency": 1415.958, "stderr": 2.069, "cost_per_test": 0.055896, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 86.891, "latency": 1498.17, "stderr": 2.069, "cost_per_test": 1.41634, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 86.517, "latency": 749.656, "stderr": 2.094, "cost_per_test": 2.138974, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.2": { "accuracy": 86.517, "latency": 1443.394, "stderr": 2.094, "cost_per_test": 2.117782, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 86.517, "latency": 2495.782, "stderr": 2.094, "cost_per_test": 4.394844, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 86.142, "latency": 2867.042, "stderr": 2.118, "cost_per_test": 19.378797, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 85.768, "latency": 854.906, "stderr": 2.142, "cost_per_test": 1.378443, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 85.768, "latency": 1335.276, "stderr": 2.142, "cost_per_test": 9.535296, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 85.768, "latency": 1914.198, "stderr": 2.142, "cost_per_test": 8.481032, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 85.393, "latency": 1137.476, "stderr": 2.165, "cost_per_test": 0.166693, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 85.019, "latency": 356.668, "stderr": 2.188, "cost_per_test": 0.04251, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 85.019, "latency": 580.87, "stderr": 2.188, "cost_per_test": 0.103181, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 84.644, "latency": 344.866, "stderr": 2.211, "cost_per_test": 0.597517, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-sol": { "accuracy": 84.644, "latency": 824.628, "stderr": 2.211, "cost_per_test": 4.460204, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 83.895, "latency": 908.618, "stderr": 2.254, "cost_per_test": 7.56413, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 82.022, "latency": 1227.488, "stderr": 2.354, "cost_per_test": 0.345901, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.5": { "accuracy": 81.648, "latency": 596.268, "stderr": 2.373, "cost_per_test": 6.932258, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 81.273, "latency": 1024.006, "stderr": 2.392, "cost_per_test": 3.252605, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 80.524, "latency": 522.006, "stderr": 2.428, "cost_per_test": 2.378166, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 80.15, "latency": 452.15, "stderr": 2.446, "cost_per_test": 0.035311, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 79.775, "latency": 184.048, "stderr": 2.463, "cost_per_test": 0.051964, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 79.026, "latency": 3576.986, "stderr": 2.496, "cost_per_test": 0.539041, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 76.779, "latency": 941.828, "stderr": 2.589, "cost_per_test": 9.61742, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 74.906, "latency": 104.598, "stderr": 2.658, "cost_per_test": 0.356731, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 70.037, "latency": 427.864, "stderr": 2.809, "cost_per_test": 0.431199, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "zai/glm-5.1": { "accuracy": 69.288, "latency": 488.464, "stderr": 2.828, "cost_per_test": 0.447406, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.3": { "accuracy": 67.416, "latency": 174.518, "stderr": 2.874, "cost_per_test": 0.413946, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 67.041, "latency": 503.798, "stderr": 2.882, "cost_per_test": 0.557082, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 67.041, "latency": 1624.24, "stderr": 2.882, "cost_per_test": 0.908831, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 65.543, "latency": 360.998, "stderr": 2.914, "cost_per_test": 0.145792, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 65.169, "latency": 1585.042, "stderr": 2.921, "cost_per_test": 0.137375, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 65.169, "latency": 5938.524, "stderr": 2.921, "cost_per_test": 41.44865, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 61.049, "latency": 3775.334, "stderr": 2.99, "cost_per_test": 5.593312, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 59.551, "latency": 1137.594, "stderr": 3.009, "cost_per_test": 1.209207, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 57.303, "latency": 1390.64, "stderr": 3.033, "cost_per_test": 0.302625, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 56.682, "latency": 582.58, "stderr": 3.372, "cost_per_test": 0.18267, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 56.554, "latency": 476.506, "stderr": 3.039, "cost_per_test": 0.721268, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 50.936, "latency": 2956.134, "stderr": 3.065, "cost_per_test": 13.912957, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "poolside/laguna-m.1": { "accuracy": 49.749, "latency": 476.208, "stderr": 3.553, "cost_per_test": 0.424558, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 48.315, "latency": 2109.852, "stderr": 3.064, "cost_per_test": 3.610666, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 47.94, "latency": 3186.104, "stderr": 3.063, "cost_per_test": 21.148726, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 47.191, "latency": 946.888, "stderr": 3.061, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 46.816, "latency": 162.686, "stderr": 3.059, "cost_per_test": 0.183489, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 46.231, "latency": 772.505, "stderr": 3.543, "cost_per_test": 1.156532, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 42.523, "latency": 1524.202, "stderr": 3.387, "cost_per_test": 0.261866, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 40.449, "latency": 684.738, "stderr": 3.009, "cost_per_test": 0.024251, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 38.951, "latency": 821.022, "stderr": 2.99, "cost_per_test": 0.997521, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 31.461, "latency": 130.01, "stderr": 2.847, "cost_per_test": 0.199512, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 27.715, "latency": 57.228, "stderr": 2.744, "cost_per_test": 0.081949, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "criteria_pass_rate__real_estate": { "meta/muse_spark_1_3_max": { "accuracy": 96.501, "latency": 853.466, "stderr": 0.994, "cost_per_test": 2.754558, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 96.501, "latency": 1681.804, "stderr": 0.994, "cost_per_test": 2.7908, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 95.918, "latency": 3512.384, "stderr": 1.07, "cost_per_test": 16.654201, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 95.627, "latency": 934.998, "stderr": 1.106, "cost_per_test": 0.95827, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 95.044, "latency": 2073.896, "stderr": 1.174, "cost_per_test": 3.122275, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 94.752, "latency": 2428.202, "stderr": 1.206, "cost_per_test": 4.477501, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "fireworks/ember-1": { "accuracy": 94.461, "latency": 3661.252, "stderr": 1.237, "cost_per_test": 4.367429, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 94.169, "latency": 1388.648, "stderr": 1.267, "cost_per_test": 5.113504, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 94.169, "latency": 2298.07, "stderr": 1.267, "cost_per_test": 9.655661, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 93.878, "latency": 1616.44, "stderr": 1.296, "cost_per_test": 25.792468, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 93.878, "latency": 3242.052, "stderr": 1.296, "cost_per_test": 4.710728, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 93.878, "latency": 8514.508, "stderr": 1.296, "cost_per_test": 54.321067, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 93.586, "latency": 625.732, "stderr": 1.325, "cost_per_test": 1.767673, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 93.294, "latency": 1678.198, "stderr": 1.352, "cost_per_test": 21.067576, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 93.294, "latency": 2412.168, "stderr": 1.352, "cost_per_test": 5.007137, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 93.003, "latency": 1656.608, "stderr": 1.379, "cost_per_test": 1.096893, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 93.003, "latency": 2122.326, "stderr": 1.379, "cost_per_test": 0.266735, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 93.003, "latency": 3945.164, "stderr": 1.379, "cost_per_test": 28.957365, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.5": { "accuracy": 92.711, "latency": 799.052, "stderr": 1.406, "cost_per_test": 5.565429, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 92.711, "latency": 1817.306, "stderr": 1.406, "cost_per_test": 1.123838, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 92.128, "latency": 381.558, "stderr": 1.456, "cost_per_test": 2.814369, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-sol": { "accuracy": 92.128, "latency": 1000.01, "stderr": 1.456, "cost_per_test": 4.665221, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 91.837, "latency": 1153.204, "stderr": 1.481, "cost_per_test": 1.899249, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 91.837, "latency": 1535.996, "stderr": 1.481, "cost_per_test": 0.19941, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 91.545, "latency": 2259.924, "stderr": 1.504, "cost_per_test": 4.5574, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 91.545, "latency": 3422.408, "stderr": 1.504, "cost_per_test": 2.96409, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-luna": { "accuracy": 91.254, "latency": 1155.776, "stderr": 1.528, "cost_per_test": 0.51747, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 91.254, "latency": 1190.406, "stderr": 1.528, "cost_per_test": 0.101049, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 90.962, "latency": 1485.644, "stderr": 1.55, "cost_per_test": 10.731391, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 90.671, "latency": 417.46, "stderr": 1.573, "cost_per_test": 0.193926, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 90.671, "latency": 699.528, "stderr": 1.573, "cost_per_test": 0.486947, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 90.671, "latency": 2252.274, "stderr": 1.573, "cost_per_test": 0.414459, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 90.379, "latency": 1116.426, "stderr": 1.595, "cost_per_test": 2.958384, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 90.087, "latency": 841.228, "stderr": 1.616, "cost_per_test": 3.880679, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 90.087, "latency": 1090.508, "stderr": 1.616, "cost_per_test": 4.771601, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 90.087, "latency": 1886.13, "stderr": 1.616, "cost_per_test": 1.775065, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 89.504, "latency": 939.142, "stderr": 1.657, "cost_per_test": 0.066263, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 89.213, "latency": 415.102, "stderr": 1.677, "cost_per_test": 0.804107, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 88.921, "latency": 1046.09, "stderr": 1.697, "cost_per_test": 3.508749, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 88.921, "latency": 1568.462, "stderr": 1.697, "cost_per_test": 10.93977, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 88.338, "latency": 1101.184, "stderr": 1.736, "cost_per_test": 0.893435, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 86.88, "latency": 2920.534, "stderr": 1.826, "cost_per_test": 10.465474, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 86.297, "latency": 501.908, "stderr": 1.859, "cost_per_test": 2.425857, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 85.714, "latency": 589.834, "stderr": 1.892, "cost_per_test": 0.127716, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 84.548, "latency": 414.664, "stderr": 1.954, "cost_per_test": 0.040377, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 81.924, "latency": 1936.218, "stderr": 2.081, "cost_per_test": 1.163256, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 81.633, "latency": 188.604, "stderr": 2.094, "cost_per_test": 0.060815, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 80.758, "latency": 1012.23, "stderr": 2.132, "cost_per_test": 4.677799, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 79.3, "latency": 1010.134, "stderr": 2.191, "cost_per_test": 2.121529, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 79.009, "latency": 367.182, "stderr": 2.202, "cost_per_test": 0.050466, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 79.009, "latency": 1732.598, "stderr": 2.202, "cost_per_test": 12.830219, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 77.551, "latency": 1414.578, "stderr": 2.256, "cost_per_test": 0.122737, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 76.093, "latency": 1362.7, "stderr": 2.306, "cost_per_test": 1.394793, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5.3": { "accuracy": 74.344, "latency": 1969.466, "stderr": 2.362, "cost_per_test": 3.571062, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.3": { "accuracy": 70.262, "latency": 217.598, "stderr": 2.472, "cost_per_test": 0.312805, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 68.462, "latency": 751.535, "stderr": 4.091, "cost_per_test": 0.96877, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 66.472, "latency": 1430.388, "stderr": 2.553, "cost_per_test": 0.366047, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 66.045, "latency": 432.99, "stderr": 2.898, "cost_per_test": 0.200928, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 65.598, "latency": 415.564, "stderr": 2.569, "cost_per_test": 0.175367, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 64.14, "latency": 397.494, "stderr": 2.593, "cost_per_test": 0.444015, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 60.933, "latency": 2101.336, "stderr": 2.638, "cost_per_test": 4.008978, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 59.767, "latency": 546.054, "stderr": 2.652, "cost_per_test": 3.068252, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 59.767, "latency": 3620.802, "stderr": 2.652, "cost_per_test": 4.871351, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 59.475, "latency": 516.24, "stderr": 2.655, "cost_per_test": 0.54794, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "poolside/laguna-m.1": { "accuracy": 56.851, "latency": 406.892, "stderr": 2.678, "cost_per_test": 0.50823, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 53.644, "latency": 1700.696, "stderr": 2.696, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "zai/glm-5.1": { "accuracy": 49.854, "latency": 391.026, "stderr": 2.704, "cost_per_test": 0.461787, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 48.98, "latency": 2942.99, "stderr": 2.703, "cost_per_test": 21.779478, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 48.105, "latency": 387.812, "stderr": 2.702, "cost_per_test": 0.437245, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 46.939, "latency": 262.84, "stderr": 2.699, "cost_per_test": 0.886902, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 45.19, "latency": 286.224, "stderr": 2.691, "cost_per_test": 0.203965, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 44.606, "latency": 285.042, "stderr": 2.688, "cost_per_test": 0.33031, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 41.108, "latency": 802.4, "stderr": 2.661, "cost_per_test": 0.362861, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 36.443, "latency": 3371.66, "stderr": 2.602, "cost_per_test": 16.877102, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 29.446, "latency": 82.088, "stderr": 2.465, "cost_per_test": 0.20807, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 20.408, "latency": 2771.174, "stderr": 2.179, "cost_per_test": 0.038282, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } }, "criteria_pass_rate__structured_finance_securitization": { "meta/muse_spark_1_3_max": { "accuracy": 91.111, "latency": 532.566, "stderr": 1.606, "cost_per_test": 1.638501, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 90.794, "latency": 2122.848, "stderr": 1.632, "cost_per_test": 3.845931, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 90.159, "latency": 772.732, "stderr": 1.681, "cost_per_test": 2.165683, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 88.889, "latency": 1102.334, "stderr": 1.774, "cost_per_test": 1.625328, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 88.571, "latency": 2318.596, "stderr": 1.795, "cost_per_test": 9.939067, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 88.254, "latency": 3445.546, "stderr": 1.817, "cost_per_test": 13.409945, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 88.254, "latency": 6243.388, "stderr": 1.817, "cost_per_test": 41.180587, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 87.302, "latency": 565.392, "stderr": 1.879, "cost_per_test": 0.630492, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 87.302, "latency": 1580.746, "stderr": 1.879, "cost_per_test": 17.134562, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 86.984, "latency": 678.652, "stderr": 1.899, "cost_per_test": 2.18141, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3": { "accuracy": 86.984, "latency": 2851.368, "stderr": 1.899, "cost_per_test": 4.509767, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 86.667, "latency": 2279.76, "stderr": 1.918, "cost_per_test": 0.212322, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 86.349, "latency": 1695.05, "stderr": 1.938, "cost_per_test": 0.84184, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 86.032, "latency": 2674.492, "stderr": 1.956, "cost_per_test": 2.178978, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 86.032, "latency": 3180.94, "stderr": 1.956, "cost_per_test": 0.655247, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 85.714, "latency": 519.34, "stderr": 1.975, "cost_per_test": 3.636689, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 85.714, "latency": 870.666, "stderr": 1.975, "cost_per_test": 1.692267, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/ember-1": { "accuracy": 85.714, "latency": 2815.95, "stderr": 1.975, "cost_per_test": 3.196836, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 85.397, "latency": 1320.054, "stderr": 1.993, "cost_per_test": 0.163846, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 85.397, "latency": 1649.426, "stderr": 1.993, "cost_per_test": 0.092304, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 85.397, "latency": 1767.882, "stderr": 1.993, "cost_per_test": 3.302349, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.5": { "accuracy": 85.079, "latency": 578.878, "stderr": 2.011, "cost_per_test": 1.902953, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 84.762, "latency": 783.044, "stderr": 2.028, "cost_per_test": 0.055487, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 84.762, "latency": 3725.94, "stderr": 2.028, "cost_per_test": 27.562306, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "stepfun/step-5-preview": { "accuracy": 84.444, "latency": 1816.484, "stderr": 2.045, "cost_per_test": 0.983278, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "openai/gpt-6-astra": { "accuracy": 83.81, "latency": 1461.506, "stderr": 2.079, "cost_per_test": 23.080257, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 83.81, "latency": 2423.536, "stderr": 2.079, "cost_per_test": 3.983856, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.2": { "accuracy": 83.492, "latency": 1144.854, "stderr": 2.095, "cost_per_test": 1.925613, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 82.54, "latency": 564.454, "stderr": 2.142, "cost_per_test": 0.186047, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 82.331, "latency": 2118.808, "stderr": 2.343, "cost_per_test": 3.463381, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 82.222, "latency": 393.294, "stderr": 2.158, "cost_per_test": 0.786036, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 81.587, "latency": 651.854, "stderr": 2.187, "cost_per_test": 0.424212, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 81.27, "latency": 925.634, "stderr": 2.202, "cost_per_test": 1.250112, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 81.27, "latency": 1487.61, "stderr": 2.202, "cost_per_test": 11.288588, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 80.952, "latency": 484.552, "stderr": 2.216, "cost_per_test": 0.11464, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 80.952, "latency": 1117.962, "stderr": 2.216, "cost_per_test": 3.023182, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 80.952, "latency": 1148.782, "stderr": 2.216, "cost_per_test": 0.254479, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 80.952, "latency": 1313.738, "stderr": 2.216, "cost_per_test": 9.320472, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 79.897, "latency": 2994.183, "stderr": 2.885, "cost_per_test": 13.720065, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 79.365, "latency": 1252.694, "stderr": 2.284, "cost_per_test": 14.536468, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 79.365, "latency": 1812.578, "stderr": 2.284, "cost_per_test": 1.250902, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-sol": { "accuracy": 79.048, "latency": 779.112, "stderr": 2.297, "cost_per_test": 3.145583, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 78.413, "latency": 805.298, "stderr": 2.322, "cost_per_test": 1.74196, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 78.095, "latency": 1052.35, "stderr": 2.334, "cost_per_test": 3.686565, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 78.095, "latency": 1378.032, "stderr": 2.334, "cost_per_test": 5.98365, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 76.825, "latency": 365.932, "stderr": 2.381, "cost_per_test": 0.039132, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.5": { "accuracy": 76.508, "latency": 794.386, "stderr": 2.392, "cost_per_test": 7.609913, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 73.968, "latency": 2425.614, "stderr": 2.476, "cost_per_test": 4.677121, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 73.651, "latency": 752.314, "stderr": 2.486, "cost_per_test": 2.587486, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 72.063, "latency": 1934.564, "stderr": 2.532, "cost_per_test": 0.169113, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 72.063, "latency": 3112.574, "stderr": 2.532, "cost_per_test": 4.517092, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 71.429, "latency": 1699.996, "stderr": 2.549, "cost_per_test": 0.903082, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 70.794, "latency": 842.814, "stderr": 2.566, "cost_per_test": 2.506224, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 68.889, "latency": 561.12, "stderr": 2.613, "cost_per_test": 3.015746, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 67.49, "latency": 155.73, "stderr": 3.011, "cost_per_test": 0.04442, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 64.444, "latency": 206.166, "stderr": 2.701, "cost_per_test": 0.29865, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 64.127, "latency": 3641.718, "stderr": 2.707, "cost_per_test": 19.757045, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 61.905, "latency": 185.046, "stderr": 2.741, "cost_per_test": 0.388122, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 60.952, "latency": 629.124, "stderr": 2.753, "cost_per_test": 1.246746, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 60.317, "latency": 353.186, "stderr": 2.761, "cost_per_test": 0.195145, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 59.048, "latency": 663.972, "stderr": 2.775, "cost_per_test": 0.917778, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 58.848, "latency": 1394.672, "stderr": 3.163, "cost_per_test": 0.382612, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 58.025, "latency": 833.975, "stderr": 3.172, "cost_per_test": 1.118034, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 56.825, "latency": 273.01, "stderr": 2.795, "cost_per_test": 0.312857, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 55.556, "latency": 2100.302, "stderr": 2.804, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 54.286, "latency": 326.596, "stderr": 2.811, "cost_per_test": 0.038287, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "zai/glm-5.1": { "accuracy": 53.016, "latency": 385.458, "stderr": 2.817, "cost_per_test": 0.396449, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 51.746, "latency": 663.878, "stderr": 2.82, "cost_per_test": 2.811066, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 45.079, "latency": 222.714, "stderr": 2.808, "cost_per_test": 0.253469, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "inception/mercury-2.5": { "accuracy": 43.492, "latency": 79.682, "stderr": 2.798, "cost_per_test": 0.225971, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 41.753, "latency": 1494.943, "stderr": 3.55, "cost_per_test": 0.312124, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "poolside/laguna-m.1": { "accuracy": 40.317, "latency": 320.916, "stderr": 2.768, "cost_per_test": 0.381923, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 40.317, "latency": 2366.126, "stderr": 2.768, "cost_per_test": 0.022576, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 27.302, "latency": 266.99, "stderr": 2.514, "cost_per_test": 0.326243, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 26.032, "latency": 55.102, "stderr": 2.476, "cost_per_test": 0.066982, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 17.593, "latency": 145.977, "stderr": 2.597, "cost_per_test": 0.15069, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null } }, "criteria_pass_rate__tax": { "google/gemini-4-argon": { "accuracy": 93.311, "latency": 2790.458, "stderr": 1.447, "cost_per_test": 10.875562, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 92.977, "latency": 1040.15, "stderr": 1.48, "cost_per_test": 2.635161, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 92.977, "latency": 1835.168, "stderr": 1.48, "cost_per_test": 2.558771, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 91.304, "latency": 698.208, "stderr": 1.632, "cost_per_test": 0.714588, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 90.635, "latency": 892.478, "stderr": 1.688, "cost_per_test": 2.323616, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 87.625, "latency": 709.58, "stderr": 1.908, "cost_per_test": 4.513818, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 87.291, "latency": 2489.228, "stderr": 1.929, "cost_per_test": 1.290764, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 86.622, "latency": 391.522, "stderr": 1.972, "cost_per_test": 2.850433, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/ember-1": { "accuracy": 86.622, "latency": 2787.138, "stderr": 1.972, "cost_per_test": 3.914791, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 86.288, "latency": 908.22, "stderr": 1.993, "cost_per_test": 0.096872, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 85.953, "latency": 1412.566, "stderr": 2.013, "cost_per_test": 4.269538, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 85.953, "latency": 3286.746, "stderr": 2.013, "cost_per_test": 6.118001, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-5.3": { "accuracy": 85.953, "latency": 3725.868, "stderr": 2.013, "cost_per_test": 6.356447, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.2": { "accuracy": 85.619, "latency": 1637.674, "stderr": 2.033, "cost_per_test": 2.610141, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 84.95, "latency": 1470.598, "stderr": 2.071, "cost_per_test": 1.553621, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "grok/grok-4.6": { "accuracy": 84.95, "latency": 2936.336, "stderr": 2.071, "cost_per_test": 4.720799, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 84.615, "latency": 4204.678, "stderr": 2.09, "cost_per_test": 26.870915, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.5": { "accuracy": 84.281, "latency": 531.476, "stderr": 2.108, "cost_per_test": 1.829973, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 84.281, "latency": 1733.256, "stderr": 2.108, "cost_per_test": 10.87057, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 83.946, "latency": 1344.49, "stderr": 2.127, "cost_per_test": 0.247354, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 83.946, "latency": 1861.222, "stderr": 2.127, "cost_per_test": 1.092562, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 83.946, "latency": 3432.134, "stderr": 2.127, "cost_per_test": 4.824852, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 83.946, "latency": 3703.608, "stderr": 2.127, "cost_per_test": 16.089025, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 83.612, "latency": 1763.37, "stderr": 2.144, "cost_per_test": 17.075748, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 83.612, "latency": 3155.744, "stderr": 2.144, "cost_per_test": 2.991236, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 82.943, "latency": 961.292, "stderr": 2.179, "cost_per_test": 0.065423, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 82.274, "latency": 502.124, "stderr": 2.212, "cost_per_test": 0.198778, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 82.274, "latency": 2140.868, "stderr": 2.212, "cost_per_test": 10.699038, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 81.94, "latency": 2624.66, "stderr": 2.228, "cost_per_test": 8.873133, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 81.271, "latency": 1497.818, "stderr": 2.26, "cost_per_test": 0.181593, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 80.936, "latency": 1207.288, "stderr": 2.275, "cost_per_test": 5.647333, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 80.602, "latency": 1692.904, "stderr": 2.291, "cost_per_test": 9.85827, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 79.264, "latency": 390.0, "stderr": 2.349, "cost_per_test": 0.044615, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 78.93, "latency": 4209.854, "stderr": 2.362, "cost_per_test": 8.046596, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 77.926, "latency": 381.644, "stderr": 2.403, "cost_per_test": 0.767561, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 77.926, "latency": 497.872, "stderr": 2.403, "cost_per_test": 0.041821, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 77.592, "latency": 873.126, "stderr": 2.415, "cost_per_test": 2.661317, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 77.592, "latency": 1063.3, "stderr": 2.415, "cost_per_test": 1.480887, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 77.592, "latency": 2473.116, "stderr": 2.415, "cost_per_test": 1.620848, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 76.589, "latency": 1875.724, "stderr": 2.453, "cost_per_test": 0.167663, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-astra": { "accuracy": 75.585, "latency": 1655.896, "stderr": 2.488, "cost_per_test": 47.22613, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 75.585, "latency": 1716.128, "stderr": 2.488, "cost_per_test": 0.890846, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 75.585, "latency": 8351.112, "stderr": 2.488, "cost_per_test": 47.644121, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 75.251, "latency": 574.202, "stderr": 2.5, "cost_per_test": 0.106054, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 75.251, "latency": 964.33, "stderr": 2.5, "cost_per_test": 0.358568, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 74.916, "latency": 794.508, "stderr": 2.511, "cost_per_test": 0.596364, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 74.582, "latency": 530.32, "stderr": 2.522, "cost_per_test": 2.60193, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.5": { "accuracy": 72.575, "latency": 671.384, "stderr": 2.584, "cost_per_test": 4.83132, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-sol": { "accuracy": 72.575, "latency": 860.462, "stderr": 2.584, "cost_per_test": 4.446485, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 70.903, "latency": 1159.824, "stderr": 2.631, "cost_per_test": 10.025086, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 70.903, "latency": 1392.034, "stderr": 2.631, "cost_per_test": 3.796478, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.1": { "accuracy": 67.893, "latency": 456.764, "stderr": 2.705, "cost_per_test": 0.530065, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "grok/grok-4.3": { "accuracy": 63.211, "latency": 179.408, "stderr": 2.793, "cost_per_test": 0.380436, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 63.211, "latency": 2045.018, "stderr": 2.793, "cost_per_test": 0.403124, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 62.876, "latency": 231.336, "stderr": 2.799, "cost_per_test": 0.251258, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 61.873, "latency": 141.542, "stderr": 2.814, "cost_per_test": 0.039851, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 61.204, "latency": 432.008, "stderr": 2.823, "cost_per_test": 0.533038, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 60.87, "latency": 404.446, "stderr": 2.827, "cost_per_test": 0.177002, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 60.201, "latency": 394.79, "stderr": 2.836, "cost_per_test": 0.628204, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 58.528, "latency": 3328.596, "stderr": 2.854, "cost_per_test": 0.714506, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 57.525, "latency": 913.98, "stderr": 2.863, "cost_per_test": 3.459839, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 57.43, "latency": 3256.682, "stderr": 3.14, "cost_per_test": 17.274333, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "poolside/laguna-m.1": { "accuracy": 56.522, "latency": 592.95, "stderr": 2.872, "cost_per_test": 0.670951, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 56.225, "latency": 2605.465, "stderr": 3.15, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 50.502, "latency": 313.794, "stderr": 2.896, "cost_per_test": 0.24687, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 49.498, "latency": 547.97, "stderr": 2.896, "cost_per_test": 2.239645, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 49.398, "latency": 1979.525, "stderr": 3.175, "cost_per_test": 2.288379, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 48.829, "latency": 837.746, "stderr": 2.896, "cost_per_test": 0.915144, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 44.147, "latency": 3939.954, "stderr": 2.877, "cost_per_test": 0.060202, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 43.662, "latency": 720.268, "stderr": 3.406, "cost_per_test": 0.923588, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 38.796, "latency": 5267.096, "stderr": 2.823, "cost_per_test": 6.478837, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 38.127, "latency": 300.574, "stderr": 2.814, "cost_per_test": 0.312585, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 33.445, "latency": 61.968, "stderr": 2.733, "cost_per_test": 0.08909, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-2.5": { "accuracy": 27.425, "latency": 75.256, "stderr": 2.584, "cost_per_test": 0.179616, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 25.581, "latency": 778.23, "stderr": 4.733, "cost_per_test": 0.297344, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 18.06, "latency": 662.032, "stderr": 2.228, "cost_per_test": 0.805207, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null } }, "criteria_pass_rate__trusts_estates_private_client": { "meta/muse_spark_1_3_max": { "accuracy": 93.893, "latency": 1037.778, "stderr": 1.482, "cost_per_test": 1.784859, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 93.511, "latency": 1314.758, "stderr": 1.525, "cost_per_test": 1.805781, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 93.511, "latency": 3234.098, "stderr": 1.525, "cost_per_test": 16.183565, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.6": { "accuracy": 92.748, "latency": 2647.598, "stderr": 1.605, "cost_per_test": 4.653439, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 92.748, "latency": 2718.166, "stderr": 1.605, "cost_per_test": 4.184536, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 92.748, "latency": 3045.714, "stderr": 1.605, "cost_per_test": 5.517077, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.7": { "accuracy": 92.366, "latency": 2788.88, "stderr": 1.644, "cost_per_test": 12.4848, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 91.985, "latency": 1662.664, "stderr": 1.681, "cost_per_test": 18.186942, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 91.603, "latency": 613.88, "stderr": 1.717, "cost_per_test": 0.719941, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.5": { "accuracy": 90.84, "latency": 613.212, "stderr": 1.786, "cost_per_test": 1.84661, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 90.84, "latency": 951.492, "stderr": 1.786, "cost_per_test": 0.088474, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 90.84, "latency": 1659.25, "stderr": 1.786, "cost_per_test": 0.915369, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 90.84, "latency": 3696.17, "stderr": 1.786, "cost_per_test": 2.595506, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k3": { "accuracy": 90.076, "latency": 810.77, "stderr": 1.851, "cost_per_test": 4.099425, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "fireworks/ember-1": { "accuracy": 90.076, "latency": 3477.792, "stderr": 1.851, "cost_per_test": 4.018375, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 89.855, "latency": 1181.327, "stderr": 2.104, "cost_per_test": 0.059982, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 89.695, "latency": 2018.478, "stderr": 1.882, "cost_per_test": 3.714805, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 89.313, "latency": 1292.3, "stderr": 1.912, "cost_per_test": 0.163816, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 89.313, "latency": 1349.698, "stderr": 1.912, "cost_per_test": 11.396437, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 88.931, "latency": 1249.462, "stderr": 1.942, "cost_per_test": 0.23816, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "stepfun/step-5-preview": { "accuracy": 88.931, "latency": 1270.302, "stderr": 1.942, "cost_per_test": 0.74451, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 88.55, "latency": 530.524, "stderr": 1.971, "cost_per_test": 2.192393, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.2": { "accuracy": 88.55, "latency": 1109.494, "stderr": 1.971, "cost_per_test": 2.162434, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 88.168, "latency": 436.982, "stderr": 1.999, "cost_per_test": 0.15293, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 88.168, "latency": 462.122, "stderr": 1.999, "cost_per_test": 0.765564, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 88.168, "latency": 1477.192, "stderr": 1.999, "cost_per_test": 23.645844, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 87.786, "latency": 351.656, "stderr": 2.027, "cost_per_test": 2.37985, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 87.786, "latency": 985.068, "stderr": 2.027, "cost_per_test": 2.033707, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 87.786, "latency": 1340.898, "stderr": 2.027, "cost_per_test": 9.820962, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 87.786, "latency": 1942.364, "stderr": 2.027, "cost_per_test": 2.935175, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 87.405, "latency": 8870.548, "stderr": 2.054, "cost_per_test": 47.297443, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 87.023, "latency": 3221.54, "stderr": 2.08, "cost_per_test": 9.687818, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 86.641, "latency": 3106.68, "stderr": 2.106, "cost_per_test": 24.97725, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 86.26, "latency": 1368.64, "stderr": 2.131, "cost_per_test": 8.582153, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 85.878, "latency": 952.646, "stderr": 2.156, "cost_per_test": 3.743606, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-sol": { "accuracy": 85.496, "latency": 774.15, "stderr": 2.18, "cost_per_test": 2.72662, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 85.496, "latency": 1796.99, "stderr": 2.18, "cost_per_test": 2.467191, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 85.115, "latency": 1290.308, "stderr": 2.203, "cost_per_test": 2.740693, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 84.351, "latency": 485.3, "stderr": 2.249, "cost_per_test": 0.041019, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 83.969, "latency": 1151.496, "stderr": 2.271, "cost_per_test": 2.752136, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 83.588, "latency": 1459.198, "stderr": 2.293, "cost_per_test": 1.392685, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 82.061, "latency": 2344.854, "stderr": 2.375, "cost_per_test": 1.287155, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-luna": { "accuracy": 81.679, "latency": 917.72, "stderr": 2.394, "cost_per_test": 0.198018, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 81.298, "latency": 505.312, "stderr": 2.414, "cost_per_test": 0.25343, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 81.298, "latency": 883.526, "stderr": 2.414, "cost_per_test": 3.626126, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 80.916, "latency": 2288.172, "stderr": 2.432, "cost_per_test": 0.177119, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 80.534, "latency": 1068.532, "stderr": 2.451, "cost_per_test": 5.065086, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse_spark_1_3": { "accuracy": 78.626, "latency": 979.01, "stderr": 2.538, "cost_per_test": 1.596615, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 77.358, "latency": 753.492, "stderr": 2.881, "cost_per_test": 0.887587, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 75.954, "latency": 170.998, "stderr": 2.645, "cost_per_test": 0.038531, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 75.573, "latency": 104.812, "stderr": 2.66, "cost_per_test": 0.232718, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 75.573, "latency": 3244.644, "stderr": 2.66, "cost_per_test": 15.106108, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 75.191, "latency": 1141.15, "stderr": 2.673, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 74.809, "latency": 1441.398, "stderr": 2.687, "cost_per_test": 0.305605, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.1": { "accuracy": 74.046, "latency": 400.142, "stderr": 2.714, "cost_per_test": 0.400358, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 73.282, "latency": 1146.278, "stderr": 2.739, "cost_per_test": 0.6436, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 72.519, "latency": 371.812, "stderr": 2.763, "cost_per_test": 0.188846, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 70.992, "latency": 280.702, "stderr": 2.809, "cost_per_test": 0.035772, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 70.992, "latency": 323.742, "stderr": 2.809, "cost_per_test": 0.352488, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 70.531, "latency": 3375.65, "stderr": 3.176, "cost_per_test": 21.781833, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.3": { "accuracy": 69.847, "latency": 276.672, "stderr": 2.841, "cost_per_test": 0.974394, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 69.084, "latency": 1990.218, "stderr": 2.861, "cost_per_test": 3.065155, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 68.702, "latency": 2937.012, "stderr": 2.87, "cost_per_test": 0.541093, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "openai/gpt-5.5": { "accuracy": 67.557, "latency": 684.416, "stderr": 2.898, "cost_per_test": 4.928295, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 67.176, "latency": 995.132, "stderr": 2.907, "cost_per_test": 0.039247, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 66.794, "latency": 423.01, "stderr": 2.915, "cost_per_test": 0.467668, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 66.031, "latency": 541.676, "stderr": 2.932, "cost_per_test": 0.115297, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 65.566, "latency": 3814.938, "stderr": 3.271, "cost_per_test": 6.292927, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 63.208, "latency": 432.742, "stderr": 3.32, "cost_per_test": 0.175098, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "poolside/laguna-m.1": { "accuracy": 61.069, "latency": 471.38, "stderr": 3.018, "cost_per_test": 0.498053, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 58.397, "latency": 484.966, "stderr": 3.051, "cost_per_test": 0.526516, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 54.198, "latency": 187.458, "stderr": 3.084, "cost_per_test": 0.223928, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 51.908, "latency": 982.602, "stderr": 3.093, "cost_per_test": 0.226872, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 45.038, "latency": 840.962, "stderr": 3.08, "cost_per_test": 2.21818, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 41.985, "latency": 81.35, "stderr": 3.055, "cost_per_test": 0.209388, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 35.496, "latency": 45.062, "stderr": 2.962, "cost_per_test": 0.043684, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null } }, "criteria_pass_rate__white_collar_defense_investigations": { "meta/muse_spark_1_3": { "accuracy": 94.276, "latency": 1094.13, "stderr": 1.35, "cost_per_test": 2.939674, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse_spark_1_3_max": { "accuracy": 93.939, "latency": 665.522, "stderr": 1.387, "cost_per_test": 1.635544, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 93.939, "latency": 2215.2, "stderr": 1.387, "cost_per_test": 10.940583, "temperature": 1.0, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse_spark_1_2": { "accuracy": 93.266, "latency": 1278.39, "stderr": 1.457, "cost_per_test": 1.579724, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.6": { "accuracy": 92.929, "latency": 3257.702, "stderr": 1.49, "cost_per_test": 3.343428, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "grok/grok-4.7": { "accuracy": 91.919, "latency": 2354.074, "stderr": 1.584, "cost_per_test": 10.468913, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 90.572, "latency": 630.426, "stderr": 1.698, "cost_per_test": 4.016727, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.5": { "accuracy": 90.572, "latency": 708.492, "stderr": 1.698, "cost_per_test": 2.297215, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 90.572, "latency": 2273.904, "stderr": 1.698, "cost_per_test": 3.663112, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-27b": { "accuracy": 90.236, "latency": 896.148, "stderr": 1.725, "cost_per_test": 3.236421, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse_spark_1_1": { "accuracy": 89.562, "latency": 532.962, "stderr": 1.777, "cost_per_test": 0.739764, "temperature": null, "top_p": null, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 89.562, "latency": 1020.596, "stderr": 1.777, "cost_per_test": 0.16563, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 89.562, "latency": 1775.954, "stderr": 1.777, "cost_per_test": 3.154232, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "minimax/MiniMax-M3": { "accuracy": 88.552, "latency": 1106.794, "stderr": 1.851, "cost_per_test": 1.117058, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 512000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "alibaba/qwen3.8-max": { "accuracy": 88.552, "latency": 1720.744, "stderr": 1.851, "cost_per_test": 2.154821, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 87.879, "latency": 2041.394, "stderr": 1.897, "cost_per_test": 3.121405, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-4-7": { "accuracy": 87.542, "latency": 1340.046, "stderr": 1.919, "cost_per_test": 10.000136, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 87.542, "latency": 1512.602, "stderr": 1.919, "cost_per_test": 23.924481, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-fable-5": { "accuracy": 87.205, "latency": 1555.02, "stderr": 1.942, "cost_per_test": 16.626886, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "fireworks/ember-1": { "accuracy": 87.205, "latency": 2829.624, "stderr": 1.942, "cost_per_test": 2.643192, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "stepfun/step-5-preview": { "accuracy": 86.532, "latency": 1482.306, "stderr": 1.984, "cost_per_test": 0.835096, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 1024000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Stepfun", "harness": null }, "deepseek/deepseek-v4-flash-0731": { "accuracy": 86.195, "latency": 654.63, "stderr": 2.005, "cost_per_test": 0.053956, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash": { "accuracy": 86.195, "latency": 772.198, "stderr": 2.005, "cost_per_test": 0.146247, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 85.522, "latency": 758.704, "stderr": 2.045, "cost_per_test": 0.07138, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 85.522, "latency": 1775.898, "stderr": 2.045, "cost_per_test": 0.869928, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 85.185, "latency": 1588.488, "stderr": 2.065, "cost_per_test": 2.297079, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.7-flash": { "accuracy": 84.848, "latency": 683.694, "stderr": 2.084, "cost_per_test": 2.379245, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.2": { "accuracy": 84.848, "latency": 1118.582, "stderr": 2.084, "cost_per_test": 1.565734, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 84.848, "latency": 2107.032, "stderr": 2.084, "cost_per_test": 0.325751, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "anthropic/claude-sonnet-5": { "accuracy": 84.512, "latency": 2812.684, "stderr": 2.103, "cost_per_test": 6.176326, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-4-6": { "accuracy": 84.175, "latency": 1127.786, "stderr": 2.121, "cost_per_test": 2.783077, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": true, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "google/gemini-3.6-flash": { "accuracy": 83.838, "latency": 906.196, "stderr": 2.14, "cost_per_test": 2.137307, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5": { "accuracy": 83.838, "latency": 2613.808, "stderr": 2.14, "cost_per_test": 16.147201, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro": { "accuracy": 83.502, "latency": 389.944, "stderr": 2.157, "cost_per_test": 0.597321, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 82.828, "latency": 1069.216, "stderr": 2.192, "cost_per_test": 0.148855, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-4-8": { "accuracy": 82.492, "latency": 1200.23, "stderr": 2.209, "cost_per_test": 9.056848, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 82.155, "latency": 1489.37, "stderr": 2.226, "cost_per_test": 0.473577, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.7-max": { "accuracy": 80.808, "latency": 1864.574, "stderr": 2.289, "cost_per_test": 1.435625, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-fable-5-1": { "accuracy": 80.135, "latency": 8879.844, "stderr": 2.319, "cost_per_test": 42.363937, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 78.788, "latency": 3362.74, "stderr": 2.376, "cost_per_test": 24.223778, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "ant/ling-3.0-flash-af-rc3": { "accuracy": 78.451, "latency": 339.398, "stderr": 2.39, "cost_per_test": 0.03941, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "xiaomi/mimo-v2.5-pro": { "accuracy": 78.451, "latency": 603.842, "stderr": 2.39, "cost_per_test": 0.099168, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-5.6-sol": { "accuracy": 78.451, "latency": 1199.134, "stderr": 2.39, "cost_per_test": 10.413192, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 78.451, "latency": 1340.72, "stderr": 2.39, "cost_per_test": 0.15481, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.6-luna": { "accuracy": 78.114, "latency": 605.6, "stderr": 2.403, "cost_per_test": 0.303629, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.6-plus": { "accuracy": 76.431, "latency": 805.922, "stderr": 2.467, "cost_per_test": 3.049149, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-sol": { "accuracy": 76.094, "latency": 732.282, "stderr": 2.479, "cost_per_test": 2.683246, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.5-flash": { "accuracy": 73.737, "latency": 380.548, "stderr": 2.558, "cost_per_test": 1.771686, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-5.6-terra": { "accuracy": 73.401, "latency": 1094.404, "stderr": 2.568, "cost_per_test": 2.560117, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-5.4-2026-03-05": { "accuracy": 72.391, "latency": 806.106, "stderr": 2.599, "cost_per_test": 4.60707, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M2.7": { "accuracy": 71.717, "latency": 445.292, "stderr": 2.618, "cost_per_test": 0.588553, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 67.677, "latency": 2284.886, "stderr": 2.719, "cost_per_test": 3.632901, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "nvidia/nemotron-3-ultra-550b-a55b": { "accuracy": 67.34, "latency": 2326.0, "stderr": 2.726, "cost_per_test": null, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Nvidia", "harness": null }, "zai/glm-5.1": { "accuracy": 67.003, "latency": 590.94, "stderr": 2.733, "cost_per_test": 0.522794, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Zhipu AI", "harness": null }, "ant/ling-3.0-flash-2607": { "accuracy": 65.32, "latency": 192.164, "stderr": 2.766, "cost_per_test": 0.043819, "temperature": 0.6, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Ant", "harness": null }, "openai/gpt-5.5": { "accuracy": 65.32, "latency": 571.98, "stderr": 2.766, "cost_per_test": 3.615019, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k2.5-thinking": { "accuracy": 65.32, "latency": 2104.122, "stderr": 2.766, "cost_per_test": 0.366044, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "kimi/kimi-k2.6": { "accuracy": 64.983, "latency": 1149.156, "stderr": 2.773, "cost_per_test": 0.592721, "temperature": null, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.5-flash-lite": { "accuracy": 64.646, "latency": 282.53, "stderr": 2.779, "cost_per_test": 0.260621, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 64.646, "latency": 3657.376, "stderr": 2.779, "cost_per_test": 17.206836, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.5": { "accuracy": 63.973, "latency": 439.742, "stderr": 2.79, "cost_per_test": 0.028794, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.3": { "accuracy": 60.943, "latency": 140.312, "stderr": 2.836, "cost_per_test": 0.311913, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 59.596, "latency": 1084.252, "stderr": 2.852, "cost_per_test": 1.451903, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-5.4-mini-2026-03-17": { "accuracy": 56.725, "latency": 623.543, "stderr": 3.8, "cost_per_test": 0.976564, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.1-pro-preview": { "accuracy": 56.229, "latency": 275.234, "stderr": 2.884, "cost_per_test": 0.665432, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-medium-3.5": { "accuracy": 54.545, "latency": 1835.934, "stderr": 2.894, "cost_per_test": 5.191854, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 80000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.7-plus": { "accuracy": 46.465, "latency": 324.058, "stderr": 2.899, "cost_per_test": 0.148109, "temperature": 0.7, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "poolside/laguna-m.1": { "accuracy": 44.444, "latency": 286.138, "stderr": 2.888, "cost_per_test": 0.345186, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "openai/gpt-5.4-nano-2026-03-17": { "accuracy": 42.088, "latency": 301.998, "stderr": 2.87, "cost_per_test": 0.135545, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-2.5": { "accuracy": 41.077, "latency": 119.378, "stderr": 2.86, "cost_per_test": 0.176704, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "anthropic/claude-haiku-4-5-20251001-thinking": { "accuracy": 40.404, "latency": 307.222, "stderr": 2.852, "cost_per_test": 0.379403, "temperature": 1.0, "top_p": null, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Anthropic", "harness": null }, "google/gemini-3-flash-preview": { "accuracy": 39.057, "latency": 136.074, "stderr": 2.836, "cost_per_test": 0.243093, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.20-0309-reasoning": { "accuracy": 35.354, "latency": 713.156, "stderr": 2.779, "cost_per_test": 1.399547, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": 2000000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "poolside/laguna-xs.2": { "accuracy": 35.017, "latency": 440.762, "stderr": 2.773, "cost_per_test": 0.136331, "temperature": 1.0, "top_p": null, "max_output_tokens": 32768, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Poolside", "harness": null }, "google/gemini-3.1-flash-lite-preview": { "accuracy": 27.946, "latency": 47.606, "stderr": 2.608, "cost_per_test": 0.072372, "temperature": 1.0, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "fireworks/nemotron-lightning-3p5-30b-a3b": { "accuracy": 14.141, "latency": 174.8, "stderr": 2.025, "cost_per_test": 0.002798, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null } } } }