{ "metadata": { "benchmark": "HLAB-Vals-1.1", "slug": "hlab", "description": "Tests an agent's ability to complete legal work using documents, spreadsheets, presentations, and file-system tools", "benchmark_id": "legal_agent_benchmark", "family": "legal_agent_benchmark", "version": "1.1", "updated": "2026-10-22", "dataset_type": "public", "industry": "legal", "tasks": { "overall": "Overall", "antitrust_competition": "Antitrust Competition", "arbitration_international_dispute_resolution": "Arbitration International Dispute Resolution", "banking_finance": "Banking Finance", "bankruptcy_restructuring": "Bankruptcy Restructuring", "capital_markets": "Capital Markets", "corporate_governance": "Corporate Governance", "corporate_ma": "Corporate Ma", "data_privacy_cybersecurity": "Data Privacy Cybersecurity", "emerging_companies_venture_capital": "Emerging Companies Venture Capital", "employment_labor": "Employment Labor", "energy_natural_resources": "Energy Natural Resources", "environmental_esg": "Environmental Esg", "funds_asset_management": "Funds Asset Management", "healthcare_life_sciences": "Healthcare Life Sciences", "immigration": "Immigration", "insurance": "Insurance", "intellectual_property": "Intellectual Property", "international_trade_sanctions": "International Trade Sanctions", "litigation_dispute_resolution": "Litigation Dispute Resolution", "real_estate": "Real Estate", "structured_finance_securitization": "Structured Finance Securitization", "tax": "Tax", "trusts_estates_private_client": "Trusts Estates Private Client", "white_collar_defense_investigations": "White Collar Defense Investigations", "criteria_pass_rate": "Criteria Pass Rate", "criteria_pass_rate__antitrust_competition": "Antitrust Competition", "criteria_pass_rate__arbitration_international_dispute_resolution": "Arbitration International Dispute Resolution", "criteria_pass_rate__banking_finance": "Banking Finance", "criteria_pass_rate__bankruptcy_restructuring": "Bankruptcy Restructuring", "criteria_pass_rate__capital_markets": "Capital Markets", "criteria_pass_rate__corporate_governance": "Corporate Governance", "criteria_pass_rate__corporate_ma": "Corporate Ma", "criteria_pass_rate__data_privacy_cybersecurity": "Data Privacy Cybersecurity", "criteria_pass_rate__emerging_companies_venture_capital": "Emerging Companies Venture Capital", "criteria_pass_rate__employment_labor": "Employment Labor", "criteria_pass_rate__energy_natural_resources": "Energy Natural Resources", "criteria_pass_rate__environmental_esg": "Environmental Esg", "criteria_pass_rate__funds_asset_management": "Funds Asset Management", "criteria_pass_rate__healthcare_life_sciences": "Healthcare Life Sciences", "criteria_pass_rate__immigration": "Immigration", "criteria_pass_rate__insurance": "Insurance", "criteria_pass_rate__intellectual_property": "Intellectual Property", "criteria_pass_rate__international_trade_sanctions": "International Trade Sanctions", "criteria_pass_rate__litigation_dispute_resolution": "Litigation Dispute Resolution", "criteria_pass_rate__real_estate": "Real Estate", "criteria_pass_rate__structured_finance_securitization": "Structured Finance Securitization", "criteria_pass_rate__tax": "Tax", "criteria_pass_rate__trusts_estates_private_client": "Trusts Estates Private Client", "criteria_pass_rate__white_collar_defense_investigations": "White Collar Defense Investigations" }, "models": [ "alibaba/qwen3.8-max-internal", "anthropic/claude-haiku-5-5", "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5-5", "deepseek/deepseek-v4-pro-0813", "deepseek/deepseek-v4.1-flash-1017", "google/gemini-3.8-flash", "google/gemini-4-argon", "grok/grok-4.7", "inception/mercury-3", "kimi/kimi-k3", "meta/muse-spark-open", "meta/muse-wonder", "meta/muse-wonder-full", "minimax/MiniMax-M3.1-flash", "mistralai/mistral-large-4", "openai/gpt-6-astra", "openai/gpt-6-luna", "openai/gpt-6.1-sol", "reflection/Beam-1.0", "tencent/hy4-preview", "thinkingmachines/inkling", "thinkingmachines/inkling-small", "xiaomi/mimo-v2.6-flash", "xiaomi/mimo-v2.6-pro", "zai/glm-5.3", "zai/glm-5.3-flash" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "runner": "external", "mode": "agentic", "archived": false, "partner": true, "total_models": 27 }, "tasks": { "overall": { "google/gemini-4-argon": { "accuracy": 27.083, "latency": 1755.397, "stderr": 3.738, "cost_per_test": 10.54836, "token_totals": { "input_tokens": 812145091, "output_tokens": 18424207, "reasoning_tokens": 5211457, "cache_read_tokens": 733823568, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 25.0, "latency": 14355.684, "stderr": 3.599, "cost_per_test": 13.6316, "token_totals": { "input_tokens": 2657733432, "output_tokens": 13857658, "reasoning_tokens": 6185749, "cache_read_tokens": 1586608896, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse-wonder": { "accuracy": 23.75, "latency": 906.14, "stderr": 3.646, "cost_per_test": 7.081987, "token_totals": { "input_tokens": 1414833931, "output_tokens": 12111113, "reasoning_tokens": 1875186, "cache_read_tokens": 1374979944, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 23.333, "latency": 2649.375, "stderr": 3.547, "cost_per_test": 5.413598, "token_totals": { "input_tokens": 1349672470, "output_tokens": 13499459, "reasoning_tokens": 2307478, "cache_read_tokens": 1308495584, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 18.333, "latency": 2148.428, "stderr": 3.183, "cost_per_test": 10.746259, "token_totals": { "input_tokens": 1088222870, "output_tokens": 20549074, "reasoning_tokens": 8836231, "cache_read_tokens": 1043005952, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 17.5, "latency": 3483.562, "stderr": 3.222, "cost_per_test": 4.181392, "token_totals": { "input_tokens": 813025818, "output_tokens": 12794152, "reasoning_tokens": null, "cache_read_tokens": 459528168, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 15.417, "latency": 2950.56, "stderr": 3.063, "cost_per_test": 3.358663, "token_totals": { "input_tokens": 934205246, "output_tokens": 14880894, "reasoning_tokens": 6584092, "cache_read_tokens": 888375040, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 14.583, "latency": 1911.01, "stderr": 2.862, "cost_per_test": 3.987774, "token_totals": { "input_tokens": 1376154444, "output_tokens": 16202326, "reasoning_tokens": 7676761, "cache_read_tokens": 1332783858, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 14.167, "latency": 1328.108, "stderr": 2.787, "cost_per_test": 0.612596, "token_totals": { "input_tokens": 5524481146, "output_tokens": 25881985, "reasoning_tokens": 10520362, "cache_read_tokens": 5492827392, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 13.75, "latency": 1861.848, "stderr": 2.836, "cost_per_test": 0.216614, "token_totals": { "input_tokens": 631569981, "output_tokens": 10405619, "reasoning_tokens": 4622671, "cache_read_tokens": 604899072, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 13.333, "latency": 1342.563, "stderr": 2.821, "cost_per_test": 0.085646, "token_totals": { "input_tokens": 958015494, "output_tokens": 12423312, "reasoning_tokens": 6316080, "cache_read_tokens": 925243136, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 12.917, "latency": 2556.302, "stderr": 2.807, "cost_per_test": 4.144144, "token_totals": { "input_tokens": 887032678, "output_tokens": 10027338, "reasoning_tokens": 3826649, "cache_read_tokens": 857115099, "cache_write_tokens": 29371546 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 12.083, "latency": 1161.715, "stderr": 2.579, "cost_per_test": 3.656207, "token_totals": { "input_tokens": 1288268290, "output_tokens": 16504243, "reasoning_tokens": 6011153, "cache_read_tokens": 1232055515, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 10.833, "latency": 655.413, "stderr": 2.723, "cost_per_test": 0.802111, "token_totals": { "input_tokens": 746916941, "output_tokens": 9384470, "reasoning_tokens": 4362757, "cache_read_tokens": 726363264, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 10.417, "latency": 744.843, "stderr": 2.572, "cost_per_test": 0.323243, "token_totals": { "input_tokens": 1006432418, "output_tokens": 9841376, "reasoning_tokens": 3565890, "cache_read_tokens": 975803052, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 10.417, "latency": 4397.953, "stderr": 2.572, "cost_per_test": 16.612583, "token_totals": { "input_tokens": 7599893900, "output_tokens": 71504693, "reasoning_tokens": 47513756, "cache_read_tokens": 7328811056, "cache_write_tokens": 271040714 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 9.167, "latency": 3507.428, "stderr": 2.366, "cost_per_test": 21.37949, "token_totals": { "input_tokens": 3749613119, "output_tokens": 44884808, "reasoning_tokens": 27684473, "cache_read_tokens": 3547556427, "cache_write_tokens": 202035592 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 8.75, "latency": 1989.335, "stderr": 2.187, "cost_per_test": 0.98378, "token_totals": { "input_tokens": 1373541600, "output_tokens": 12484050, "reasoning_tokens": 4013313, "cache_read_tokens": 1314797886, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 7.5, "latency": 1497.601, "stderr": 2.265, "cost_per_test": 0.472848, "token_totals": { "input_tokens": 2730849267, "output_tokens": 20117381, "reasoning_tokens": 6681765, "cache_read_tokens": 2703378118, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-astra": { "accuracy": 7.083, "latency": 1953.362, "stderr": 1.899, "cost_per_test": 25.460295, "token_totals": { "input_tokens": 1310790488, "output_tokens": 10401199, "reasoning_tokens": 3452073, "cache_read_tokens": 1276652993, "cache_write_tokens": 34120712 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 6.25, "latency": 2425.625, "stderr": 1.83, "cost_per_test": 3.536302, "token_totals": { "input_tokens": 1227717408, "output_tokens": 11158751, "reasoning_tokens": 3901277, "cache_read_tokens": 1191414163, "cache_write_tokens": 36281197 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-3": { "accuracy": 5.0, "latency": 886.884, "stderr": 1.61, "cost_per_test": 0.188085, "token_totals": { "input_tokens": 659070144, "output_tokens": 6092886, "reasoning_tokens": 5836559, "cache_read_tokens": 505214083, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "reflection/Beam-1.0": { "accuracy": 4.583, "latency": 116.883, "stderr": 1.775, "cost_per_test": null, "token_totals": { "input_tokens": 540211238, "output_tokens": 4325040, "reasoning_tokens": 1196691, "cache_read_tokens": 509200704, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "openai/gpt-6-luna": { "accuracy": 4.167, "latency": 1305.488, "stderr": 1.629, "cost_per_test": 0.298018, "token_totals": { "input_tokens": 1322468906, "output_tokens": 12364131, "reasoning_tokens": 6724988, "cache_read_tokens": 1277646907, "cache_write_tokens": 44806375 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 4.167, "latency": 2620.795, "stderr": 1.518, "cost_per_test": 4.264376, "token_totals": { "input_tokens": 4904885383, "output_tokens": 69221549, "reasoning_tokens": 47929104, "cache_read_tokens": 4716242036, "cache_write_tokens": 188617435 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 3.75, "latency": 529.644, "stderr": 1.469, "cost_per_test": 1.252023, "token_totals": { "input_tokens": 538710546, "output_tokens": 7813620, "reasoning_tokens": null, "cache_read_tokens": 513145856, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 2.917, "latency": 455.265, "stderr": 1.362, "cost_per_test": 0.17869, "token_totals": { "input_tokens": 200497219, "output_tokens": 5802466, "reasoning_tokens": null, "cache_read_tokens": 190288768, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "antitrust_competition": { "google/gemini-4-argon": { "accuracy": 60.0, "latency": 1611.726, "stderr": 24.495, "cost_per_test": 15.535189, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 40.0, "latency": 2044.898, "stderr": 18.708, "cost_per_test": 6.005504, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 30.0, "latency": 1264.71, "stderr": 20.0, "cost_per_test": 0.534289, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder": { "accuracy": 20.0, "latency": 1023.55, "stderr": 20.0, "cost_per_test": 7.544454, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 20.0, "latency": 1997.668, "stderr": 20.0, "cost_per_test": 9.438476, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 20.0, "latency": 2261.178, "stderr": 20.0, "cost_per_test": 2.589032, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse-spark-open": { "accuracy": 20.0, "latency": 12012.07, "stderr": 20.0, "cost_per_test": 3.976357, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 10.0, "latency": 1978.794, "stderr": 10.0, "cost_per_test": 28.281218, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 10.0, "latency": 2364.752, "stderr": 10.0, "cost_per_test": 0.077418, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 10.0, "latency": 2421.532, "stderr": 10.0, "cost_per_test": 2.7654, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 10.0, "latency": 4001.192, "stderr": 10.0, "cost_per_test": 17.847147, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 104.63, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 471.488, "stderr": 0.0, "cost_per_test": 0.170509, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 654.154, "stderr": 0.0, "cost_per_test": 0.759752, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 691.274, "stderr": 0.0, "cost_per_test": 0.266224, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 694.13, "stderr": 0.0, "cost_per_test": 1.357335, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 890.898, "stderr": 0.0, "cost_per_test": 3.871498, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 1001.484, "stderr": 0.0, "cost_per_test": 0.196264, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1062.146, "stderr": 0.0, "cost_per_test": 0.226157, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 1538.864, "stderr": 0.0, "cost_per_test": 2.533451, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 0.0, "latency": 1602.082, "stderr": 0.0, "cost_per_test": 0.447689, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1690.59, "stderr": 0.0, "cost_per_test": 0.192669, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1982.912, "stderr": 0.0, "cost_per_test": 0.789632, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2274.678, "stderr": 0.0, "cost_per_test": 3.460397, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 2346.238, "stderr": 0.0, "cost_per_test": 3.732132, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3063.884, "stderr": 0.0, "cost_per_test": 23.046646, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 5773.048, "stderr": 0.0, "cost_per_test": 3.909019, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null } }, "arbitration_international_dispute_resolution": { "xiaomi/mimo-v2.6-flash": { "accuracy": 30.0, "latency": 1107.096, "stderr": 20.0, "cost_per_test": 0.098258, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-spark-open": { "accuracy": 30.0, "latency": 16647.192, "stderr": 20.0, "cost_per_test": 17.977879, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 20.0, "latency": 626.442, "stderr": 20.0, "cost_per_test": 0.933152, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 20.0, "latency": 858.63, "stderr": 20.0, "cost_per_test": 0.546196, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 1735.19, "stderr": 20.0, "cost_per_test": 0.22831, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 20.0, "latency": 1874.266, "stderr": 20.0, "cost_per_test": 12.641362, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-wonder": { "accuracy": 10.0, "latency": 842.758, "stderr": 10.0, "cost_per_test": 6.853377, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 10.0, "latency": 1273.796, "stderr": 10.0, "cost_per_test": 0.66721, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder-full": { "accuracy": 10.0, "latency": 2620.15, "stderr": 10.0, "cost_per_test": 5.114031, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 10.0, "latency": 2832.902, "stderr": 10.0, "cost_per_test": 5.231339, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 10.0, "latency": 3010.736, "stderr": 10.0, "cost_per_test": 5.265369, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 10.0, "latency": 3841.334, "stderr": 10.0, "cost_per_test": 22.280508, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 10.0, "latency": 7424.862, "stderr": 10.0, "cost_per_test": 4.415408, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 113.746, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 604.136, "stderr": 0.0, "cost_per_test": 0.213826, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 692.844, "stderr": 0.0, "cost_per_test": 0.801309, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 826.742, "stderr": 0.0, "cost_per_test": 0.220421, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 1260.926, "stderr": 0.0, "cost_per_test": 4.266851, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1371.914, "stderr": 0.0, "cost_per_test": 0.423667, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 0.0, "latency": 1490.442, "stderr": 0.0, "cost_per_test": 0.572096, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 1998.852, "stderr": 0.0, "cost_per_test": 5.120208, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 2052.194, "stderr": 0.0, "cost_per_test": 37.28026, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 2222.162, "stderr": 0.0, "cost_per_test": 12.524633, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2687.456, "stderr": 0.0, "cost_per_test": 5.097847, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 0.0, "latency": 3255.392, "stderr": 0.0, "cost_per_test": 4.767735, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 3592.504, "stderr": 0.0, "cost_per_test": 1.100251, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 5100.89, "stderr": 0.0, "cost_per_test": 16.01159, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "banking_finance": { "google/gemini-4-argon": { "accuracy": 40.0, "latency": 1393.73, "stderr": 18.708, "cost_per_test": 8.507105, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 40.0, "latency": 17997.52, "stderr": 24.495, "cost_per_test": 18.435543, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse-wonder": { "accuracy": 30.0, "latency": 936.4, "stderr": 20.0, "cost_per_test": 6.016663, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 30.0, "latency": 3124.788, "stderr": 20.0, "cost_per_test": 5.41453, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 10.0, "latency": 1273.962, "stderr": 10.0, "cost_per_test": 0.604563, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 10.0, "latency": 1802.172, "stderr": 10.0, "cost_per_test": 0.071494, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-astra": { "accuracy": 10.0, "latency": 1845.728, "stderr": 10.0, "cost_per_test": 26.702694, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 10.0, "latency": 2181.91, "stderr": 10.0, "cost_per_test": 3.430828, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 10.0, "latency": 3837.702, "stderr": 10.0, "cost_per_test": 3.38767, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 116.862, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 445.262, "stderr": 0.0, "cost_per_test": 0.169997, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 492.62, "stderr": 0.0, "cost_per_test": 1.070964, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 548.724, "stderr": 0.0, "cost_per_test": 0.087277, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 706.31, "stderr": 0.0, "cost_per_test": 0.232024, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 751.412, "stderr": 0.0, "cost_per_test": 0.76189, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 1130.544, "stderr": 0.0, "cost_per_test": 2.511915, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1218.504, "stderr": 0.0, "cost_per_test": 0.239206, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 0.0, "latency": 1557.112, "stderr": 0.0, "cost_per_test": 0.363031, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 1666.362, "stderr": 0.0, "cost_per_test": 3.316185, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1747.954, "stderr": 0.0, "cost_per_test": 0.183967, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2121.904, "stderr": 0.0, "cost_per_test": 10.337825, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 2357.686, "stderr": 0.0, "cost_per_test": 3.100539, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2593.164, "stderr": 0.0, "cost_per_test": 4.633581, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 0.0, "latency": 2841.088, "stderr": 0.0, "cost_per_test": 3.059804, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3077.896, "stderr": 0.0, "cost_per_test": 23.532694, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3454.478, "stderr": 0.0, "cost_per_test": 16.890636, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 3892.496, "stderr": 0.0, "cost_per_test": 0.858064, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null } }, "bankruptcy_restructuring": { "grok/grok-4.7": { "accuracy": 30.0, "latency": 2192.152, "stderr": 20.0, "cost_per_test": 12.833742, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 30.0, "latency": 2901.7, "stderr": 20.0, "cost_per_test": 6.272163, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 30.0, "latency": 6164.394, "stderr": 20.0, "cost_per_test": 5.241005, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 30.0, "latency": 19897.52, "stderr": 20.0, "cost_per_test": 10.090618, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse-wonder": { "accuracy": 20.0, "latency": 945.124, "stderr": 20.0, "cost_per_test": 7.317233, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 20.0, "latency": 1198.624, "stderr": 20.0, "cost_per_test": 1.003903, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 2160.01, "stderr": 20.0, "cost_per_test": 0.309228, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 10.0, "latency": 1285.442, "stderr": 10.0, "cost_per_test": 0.640206, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 10.0, "latency": 2053.988, "stderr": 10.0, "cost_per_test": 4.33963, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 10.0, "latency": 2414.9, "stderr": 10.0, "cost_per_test": 3.872355, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 10.0, "latency": 3911.966, "stderr": 10.0, "cost_per_test": 15.129321, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 96.752, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 491.618, "stderr": 0.0, "cost_per_test": 0.153217, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 773.418, "stderr": 0.0, "cost_per_test": 1.383205, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 779.568, "stderr": 0.0, "cost_per_test": 0.396682, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 969.208, "stderr": 0.0, "cost_per_test": 0.211254, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 1267.312, "stderr": 0.0, "cost_per_test": 4.332858, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1306.948, "stderr": 0.0, "cost_per_test": 0.315042, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 1549.322, "stderr": 0.0, "cost_per_test": 10.336011, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 0.0, "latency": 1586.546, "stderr": 0.0, "cost_per_test": 0.501587, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1968.368, "stderr": 0.0, "cost_per_test": 26.126342, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 2261.908, "stderr": 0.0, "cost_per_test": 1.145459, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 2674.52, "stderr": 0.0, "cost_per_test": 0.116291, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2985.2, "stderr": 0.0, "cost_per_test": 5.081934, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 0.0, "latency": 3234.16, "stderr": 0.0, "cost_per_test": 3.947315, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 3395.826, "stderr": 0.0, "cost_per_test": 5.810019, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3463.16, "stderr": 0.0, "cost_per_test": 22.051917, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "capital_markets": { "mistralai/mistral-large-4": { "accuracy": 50.0, "latency": 1449.372, "stderr": 22.361, "cost_per_test": 2.151201, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 40.0, "latency": 1403.692, "stderr": 24.495, "cost_per_test": 9.03284, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 30.0, "latency": 749.014, "stderr": 20.0, "cost_per_test": 7.059417, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 30.0, "latency": 895.892, "stderr": 20.0, "cost_per_test": 0.073201, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 30.0, "latency": 1186.85, "stderr": 20.0, "cost_per_test": 2.819389, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "tencent/hy4-preview": { "accuracy": 30.0, "latency": 1235.536, "stderr": 20.0, "cost_per_test": 0.791921, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "grok/grok-4.7": { "accuracy": 30.0, "latency": 2173.218, "stderr": 20.0, "cost_per_test": 9.948149, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 20.0, "latency": 337.696, "stderr": 20.0, "cost_per_test": 1.181787, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 20.0, "latency": 415.398, "stderr": 20.0, "cost_per_test": 0.158893, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-luna": { "accuracy": 20.0, "latency": 1059.256, "stderr": 20.0, "cost_per_test": 0.220062, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 20.0, "latency": 1270.292, "stderr": 20.0, "cost_per_test": 0.604559, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 20.0, "latency": 1505.512, "stderr": 20.0, "cost_per_test": 20.140295, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 1616.54, "stderr": 12.247, "cost_per_test": 0.197788, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": 1684.458, "stderr": 12.247, "cost_per_test": 3.571137, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 20.0, "latency": 1917.712, "stderr": 20.0, "cost_per_test": 2.741122, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 20.0, "latency": 2223.106, "stderr": 12.247, "cost_per_test": 3.753987, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-wonder-full": { "accuracy": 20.0, "latency": 2353.468, "stderr": 20.0, "cost_per_test": 4.235198, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 20.0, "latency": 3458.518, "stderr": 20.0, "cost_per_test": 14.684627, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 20.0, "latency": 7272.288, "stderr": 12.247, "cost_per_test": 11.942399, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 97.144, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 396.136, "stderr": 0.0, "cost_per_test": 0.094975, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 442.964, "stderr": 0.0, "cost_per_test": 0.628074, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 538.678, "stderr": 0.0, "cost_per_test": 0.198307, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 0.0, "latency": 1222.418, "stderr": 0.0, "cost_per_test": 0.336279, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 1822.032, "stderr": 0.0, "cost_per_test": 2.677304, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 0.0, "latency": 2232.706, "stderr": 0.0, "cost_per_test": 2.776428, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 2995.246, "stderr": 0.0, "cost_per_test": 20.393428, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "corporate_governance": { "google/gemini-4-argon": { "accuracy": 60.0, "latency": 1910.844, "stderr": 24.495, "cost_per_test": 7.288851, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 30.0, "latency": 869.038, "stderr": 20.0, "cost_per_test": 5.385755, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 20.0, "latency": 1100.274, "stderr": 12.247, "cost_per_test": 4.21486, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 20.0, "latency": 1497.44, "stderr": 20.0, "cost_per_test": 0.70205, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 1757.566, "stderr": 12.247, "cost_per_test": 0.202579, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 20.0, "latency": 2097.538, "stderr": 20.0, "cost_per_test": 0.628725, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "kimi/kimi-k3": { "accuracy": 20.0, "latency": 2396.124, "stderr": 20.0, "cost_per_test": 3.607788, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 20.0, "latency": 2713.342, "stderr": 20.0, "cost_per_test": 5.247151, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 20.0, "latency": 2735.374, "stderr": 20.0, "cost_per_test": 5.218937, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-spark-open": { "accuracy": 20.0, "latency": 17034.724, "stderr": 12.247, "cost_per_test": 26.592215, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 10.0, "latency": 507.184, "stderr": 10.0, "cost_per_test": 1.493244, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 10.0, "latency": 622.074, "stderr": 10.0, "cost_per_test": 0.264988, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 10.0, "latency": 2408.28, "stderr": 10.0, "cost_per_test": 4.104081, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 10.0, "latency": 2584.122, "stderr": 10.0, "cost_per_test": 3.108266, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 10.0, "latency": 5172.942, "stderr": 10.0, "cost_per_test": 16.225332, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 140.722, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 534.078, "stderr": 0.0, "cost_per_test": 0.200642, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 566.566, "stderr": 0.0, "cost_per_test": 0.759983, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 1100.848, "stderr": 0.0, "cost_per_test": 0.07785, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1245.05, "stderr": 0.0, "cost_per_test": 0.225592, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 1257.38, "stderr": 0.0, "cost_per_test": 0.319761, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1770.524, "stderr": 0.0, "cost_per_test": 1.030237, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1893.436, "stderr": 0.0, "cost_per_test": 28.949058, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 1900.05, "stderr": 0.0, "cost_per_test": 4.113932, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2099.78, "stderr": 0.0, "cost_per_test": 11.205219, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2758.188, "stderr": 0.0, "cost_per_test": 4.581533, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3589.944, "stderr": 0.0, "cost_per_test": 24.770063, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "corporate_ma": { "meta/muse-wonder": { "accuracy": 40.0, "latency": 894.054, "stderr": 24.495, "cost_per_test": 7.82908, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 40.0, "latency": 2372.524, "stderr": 24.495, "cost_per_test": 3.938856, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 40.0, "latency": 2751.628, "stderr": 24.495, "cost_per_test": 3.053264, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 30.0, "latency": 1001.998, "stderr": 12.247, "cost_per_test": 3.550379, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-4-argon": { "accuracy": 30.0, "latency": 1717.438, "stderr": 20.0, "cost_per_test": 8.132612, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 30.0, "latency": 2894.524, "stderr": 20.0, "cost_per_test": 5.165325, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 20.0, "latency": 369.67, "stderr": 20.0, "cost_per_test": 0.181179, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 20.0, "latency": 587.404, "stderr": 20.0, "cost_per_test": 0.785052, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 20.0, "latency": 1133.996, "stderr": 20.0, "cost_per_test": 0.07413, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 20.0, "latency": 1348.014, "stderr": 20.0, "cost_per_test": 0.64235, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 20.0, "latency": 1532.814, "stderr": 20.0, "cost_per_test": 0.274244, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 1697.334, "stderr": 20.0, "cost_per_test": 0.176755, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": 1892.662, "stderr": 20.0, "cost_per_test": 4.298734, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 20.0, "latency": 1940.384, "stderr": 12.247, "cost_per_test": 0.94797, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "grok/grok-4.7": { "accuracy": 20.0, "latency": 2005.262, "stderr": 20.0, "cost_per_test": 8.576322, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 20.0, "latency": 2093.188, "stderr": 12.247, "cost_per_test": 22.768442, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 20.0, "latency": 2164.786, "stderr": 20.0, "cost_per_test": 3.081992, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 20.0, "latency": 3267.648, "stderr": 20.0, "cost_per_test": 24.674742, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 20.0, "latency": 4191.198, "stderr": 20.0, "cost_per_test": 15.021633, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 20.0, "latency": 13513.444, "stderr": 20.0, "cost_per_test": 11.299235, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 10.0, "latency": 456.474, "stderr": 10.0, "cost_per_test": 1.157097, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 10.0, "latency": 662.998, "stderr": 10.0, "cost_per_test": 0.233836, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-3": { "accuracy": 10.0, "latency": 785.954, "stderr": 10.0, "cost_per_test": 0.208695, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 113.714, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 0.0, "latency": 1048.75, "stderr": 0.0, "cost_per_test": 0.385034, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2321.544, "stderr": 0.0, "cost_per_test": 4.04342, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2442.158, "stderr": 0.0, "cost_per_test": 3.48516, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null } }, "data_privacy_cybersecurity": { "google/gemini-4-argon": { "accuracy": 50.0, "latency": 1849.552, "stderr": 22.361, "cost_per_test": 8.325019, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 50.0, "latency": 8016.45, "stderr": 22.361, "cost_per_test": 6.234146, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 40.0, "latency": 1600.072, "stderr": 18.708, "cost_per_test": 0.204203, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 40.0, "latency": 2038.44, "stderr": 24.495, "cost_per_test": 13.898195, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 30.0, "latency": 532.25, "stderr": 20.0, "cost_per_test": 0.795228, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 30.0, "latency": 2027.386, "stderr": 20.0, "cost_per_test": 6.026136, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 30.0, "latency": 2631.3, "stderr": 20.0, "cost_per_test": 5.085316, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 30.0, "latency": 2644.256, "stderr": 20.0, "cost_per_test": 3.560927, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 30.0, "latency": 3116.616, "stderr": 20.0, "cost_per_test": 21.050628, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "inception/mercury-3": { "accuracy": 20.0, "latency": 803.208, "stderr": 12.247, "cost_per_test": 0.159261, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 20.0, "latency": 1295.87, "stderr": 20.0, "cost_per_test": 2.439638, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "tencent/hy4-preview": { "accuracy": 20.0, "latency": 1542.326, "stderr": 12.247, "cost_per_test": 1.156317, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 20.0, "latency": 1603.698, "stderr": 12.247, "cost_per_test": 0.652971, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "kimi/kimi-k3": { "accuracy": 20.0, "latency": 2086.04, "stderr": 12.247, "cost_per_test": 4.139175, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 20.0, "latency": 2553.994, "stderr": 20.0, "cost_per_test": 5.724329, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 20.0, "latency": 4080.704, "stderr": 20.0, "cost_per_test": 16.205227, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 10.0, "latency": 120.522, "stderr": 10.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling": { "accuracy": 10.0, "latency": 568.356, "stderr": 10.0, "cost_per_test": 1.636783, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 10.0, "latency": 702.086, "stderr": 10.0, "cost_per_test": 0.331272, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse-wonder": { "accuracy": 10.0, "latency": 830.088, "stderr": 10.0, "cost_per_test": 6.429998, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 10.0, "latency": 1114.628, "stderr": 10.0, "cost_per_test": 0.083103, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 10.0, "latency": 1228.252, "stderr": 10.0, "cost_per_test": 0.52666, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 10.0, "latency": 2134.04, "stderr": 10.0, "cost_per_test": 32.097478, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 401.836, "stderr": 0.0, "cost_per_test": 0.203808, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1319.848, "stderr": 0.0, "cost_per_test": 0.292544, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2352.11, "stderr": 0.0, "cost_per_test": 3.758665, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2375.93, "stderr": 0.0, "cost_per_test": 3.908605, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "emerging_companies_venture_capital": { "meta/muse-wonder": { "accuracy": 40.0, "latency": 972.262, "stderr": 24.495, "cost_per_test": 7.098804, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 30.0, "latency": 946.74, "stderr": 20.0, "cost_per_test": 3.954288, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 30.0, "latency": 20320.366, "stderr": 20.0, "cost_per_test": 18.393531, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 20.0, "latency": 618.824, "stderr": 20.0, "cost_per_test": 0.819599, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 20.0, "latency": 1103.358, "stderr": 20.0, "cost_per_test": 0.078418, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-4-argon": { "accuracy": 20.0, "latency": 1468.188, "stderr": 20.0, "cost_per_test": 13.959833, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 20.0, "latency": 2030.458, "stderr": 20.0, "cost_per_test": 20.81811, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 2035.666, "stderr": 20.0, "cost_per_test": 0.222921, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": 2229.252, "stderr": 12.247, "cost_per_test": 4.599163, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 20.0, "latency": 2231.368, "stderr": 20.0, "cost_per_test": 4.18071, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.7": { "accuracy": 20.0, "latency": 2252.824, "stderr": 12.247, "cost_per_test": 9.503463, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 20.0, "latency": 2548.358, "stderr": 20.0, "cost_per_test": 3.154588, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 20.0, "latency": 3632.004, "stderr": 20.0, "cost_per_test": 4.022647, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "inception/mercury-3": { "accuracy": 10.0, "latency": 955.938, "stderr": 10.0, "cost_per_test": 0.148034, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-6-luna": { "accuracy": 10.0, "latency": 1360.808, "stderr": 10.0, "cost_per_test": 0.274165, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 10.0, "latency": 1362.822, "stderr": 10.0, "cost_per_test": 0.990763, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 10.0, "latency": 2707.794, "stderr": 10.0, "cost_per_test": 3.778376, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 10.0, "latency": 3178.396, "stderr": 10.0, "cost_per_test": 19.621784, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 10.0, "latency": 4124.448, "stderr": 10.0, "cost_per_test": 16.606099, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 143.092, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 449.274, "stderr": 0.0, "cost_per_test": 0.200938, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 744.508, "stderr": 0.0, "cost_per_test": 1.206641, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 769.368, "stderr": 0.0, "cost_per_test": 0.28403, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 0.0, "latency": 1523.606, "stderr": 0.0, "cost_per_test": 0.437811, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 0.0, "latency": 1539.008, "stderr": 0.0, "cost_per_test": 0.633521, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "meta/muse-wonder-full": { "accuracy": 0.0, "latency": 2502.37, "stderr": 0.0, "cost_per_test": 6.394651, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2505.308, "stderr": 0.0, "cost_per_test": 4.158411, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "employment_labor": { "mistralai/mistral-large-4": { "accuracy": 10.0, "latency": 4514.362, "stderr": 10.0, "cost_per_test": 2.622256, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 123.484, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 400.446, "stderr": 0.0, "cost_per_test": 0.143202, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 474.796, "stderr": 0.0, "cost_per_test": 1.781902, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 495.416, "stderr": 0.0, "cost_per_test": 0.197427, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 631.722, "stderr": 0.0, "cost_per_test": 0.188769, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse-wonder": { "accuracy": 0.0, "latency": 862.152, "stderr": 0.0, "cost_per_test": 7.283077, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 1108.248, "stderr": 0.0, "cost_per_test": 3.168674, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 1128.258, "stderr": 0.0, "cost_per_test": 0.5861, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1239.982, "stderr": 0.0, "cost_per_test": 0.165124, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 0.0, "latency": 1291.326, "stderr": 0.0, "cost_per_test": 0.520216, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 1416.794, "stderr": 0.0, "cost_per_test": 6.38748, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1778.602, "stderr": 0.0, "cost_per_test": 14.13358, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 0.0, "latency": 1855.334, "stderr": 0.0, "cost_per_test": 0.426654, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 1861.322, "stderr": 0.0, "cost_per_test": 2.739975, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1990.538, "stderr": 0.0, "cost_per_test": 0.881184, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 2219.336, "stderr": 0.0, "cost_per_test": 0.176119, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2245.098, "stderr": 0.0, "cost_per_test": 9.137915, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2415.098, "stderr": 0.0, "cost_per_test": 2.463712, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2478.518, "stderr": 0.0, "cost_per_test": 3.640709, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 2499.984, "stderr": 0.0, "cost_per_test": 0.067578, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-wonder-full": { "accuracy": 0.0, "latency": 2510.782, "stderr": 0.0, "cost_per_test": 4.65707, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 0.0, "latency": 2745.574, "stderr": 0.0, "cost_per_test": 2.560531, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 2750.208, "stderr": 0.0, "cost_per_test": 3.430645, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3164.882, "stderr": 0.0, "cost_per_test": 14.514568, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 4200.754, "stderr": 0.0, "cost_per_test": 17.260921, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 0.0, "latency": 16944.098, "stderr": 0.0, "cost_per_test": 12.206492, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null } }, "energy_natural_resources": { "meta/muse-wonder": { "accuracy": 70.0, "latency": 1067.054, "stderr": 20.0, "cost_per_test": 11.537685, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 60.0, "latency": 3120.71, "stderr": 24.495, "cost_per_test": 6.264605, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 50.0, "latency": 621.178, "stderr": 22.361, "cost_per_test": 0.33458, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3": { "accuracy": 50.0, "latency": 1863.802, "stderr": 22.361, "cost_per_test": 5.176944, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 50.0, "latency": 3915.456, "stderr": 22.361, "cost_per_test": 26.383042, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 40.0, "latency": 548.636, "stderr": 24.495, "cost_per_test": 0.979999, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 40.0, "latency": 1434.758, "stderr": 24.495, "cost_per_test": 0.616064, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "grok/grok-4.7": { "accuracy": 40.0, "latency": 1822.396, "stderr": 18.708, "cost_per_test": 11.371091, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 40.0, "latency": 2350.706, "stderr": 24.495, "cost_per_test": 14.055481, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 40.0, "latency": 3576.094, "stderr": 24.495, "cost_per_test": 7.569589, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 40.0, "latency": 17549.494, "stderr": 24.495, "cost_per_test": 22.510187, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 30.0, "latency": 1272.778, "stderr": 20.0, "cost_per_test": 0.69079, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 30.0, "latency": 1398.494, "stderr": 20.0, "cost_per_test": 0.576369, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 30.0, "latency": 1592.142, "stderr": 20.0, "cost_per_test": 0.125628, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 30.0, "latency": 2322.328, "stderr": 20.0, "cost_per_test": 1.207559, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-astra": { "accuracy": 30.0, "latency": 2434.558, "stderr": 20.0, "cost_per_test": 52.676518, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 30.0, "latency": 2751.118, "stderr": 20.0, "cost_per_test": 5.971747, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 30.0, "latency": 3212.248, "stderr": 20.0, "cost_per_test": 4.853818, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "reflection/Beam-1.0": { "accuracy": 20.0, "latency": 101.376, "stderr": 20.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 20.0, "latency": 1422.04, "stderr": 20.0, "cost_per_test": 4.405884, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 1924.146, "stderr": 20.0, "cost_per_test": 0.288516, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 20.0, "latency": 2661.574, "stderr": 20.0, "cost_per_test": 3.699196, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 10.0, "latency": 318.21, "stderr": 10.0, "cost_per_test": 0.186124, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-3": { "accuracy": 10.0, "latency": 1050.646, "stderr": 10.0, "cost_per_test": 0.277782, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 10.0, "latency": 2951.16, "stderr": 10.0, "cost_per_test": 5.570413, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 231.65, "stderr": 0.0, "cost_per_test": 0.939334, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 5117.248, "stderr": 0.0, "cost_per_test": 17.609237, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "environmental_esg": { "meta/muse-wonder": { "accuracy": 20.0, "latency": 820.33, "stderr": 20.0, "cost_per_test": 5.178019, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 20.0, "latency": 1868.158, "stderr": 12.247, "cost_per_test": 8.278315, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 20.0, "latency": 2908.808, "stderr": 20.0, "cost_per_test": 4.77579, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 10.0, "latency": 861.556, "stderr": 10.0, "cost_per_test": 0.300222, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 145.024, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 506.562, "stderr": 0.0, "cost_per_test": 0.17309, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 535.646, "stderr": 0.0, "cost_per_test": 1.189505, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 678.776, "stderr": 0.0, "cost_per_test": 0.727139, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 1093.524, "stderr": 0.0, "cost_per_test": 0.24568, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 1119.692, "stderr": 0.0, "cost_per_test": 0.086836, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1205.648, "stderr": 0.0, "cost_per_test": 0.156612, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 1210.54, "stderr": 0.0, "cost_per_test": 3.219091, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 0.0, "latency": 1433.79, "stderr": 0.0, "cost_per_test": 0.635298, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 0.0, "latency": 1706.268, "stderr": 0.0, "cost_per_test": 0.440838, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1835.898, "stderr": 0.0, "cost_per_test": 0.897665, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1976.582, "stderr": 0.0, "cost_per_test": 17.922383, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 2278.048, "stderr": 0.0, "cost_per_test": 0.232534, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2350.754, "stderr": 0.0, "cost_per_test": 10.857757, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 2625.67, "stderr": 0.0, "cost_per_test": 3.885641, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2681.314, "stderr": 0.0, "cost_per_test": 3.292441, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 2824.202, "stderr": 0.0, "cost_per_test": 3.651165, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2880.8, "stderr": 0.0, "cost_per_test": 4.552595, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 3014.468, "stderr": 0.0, "cost_per_test": 3.563589, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 0.0, "latency": 3321.142, "stderr": 0.0, "cost_per_test": 2.910771, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3643.502, "stderr": 0.0, "cost_per_test": 24.257857, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 4846.184, "stderr": 0.0, "cost_per_test": 18.079139, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 0.0, "latency": 9213.876, "stderr": 0.0, "cost_per_test": 12.590654, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null } }, "funds_asset_management": { "google/gemini-4-argon": { "accuracy": 60.0, "latency": 2070.686, "stderr": 18.708, "cost_per_test": 10.601161, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 60.0, "latency": 18279.368, "stderr": 24.495, "cost_per_test": 14.222782, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 40.0, "latency": 1356.48, "stderr": 18.708, "cost_per_test": 0.654524, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.7": { "accuracy": 40.0, "latency": 2190.716, "stderr": 24.495, "cost_per_test": 12.657496, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 40.0, "latency": 3311.582, "stderr": 24.495, "cost_per_test": 6.091472, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "reflection/Beam-1.0": { "accuracy": 30.0, "latency": 125.446, "stderr": 20.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 30.0, "latency": 1378.846, "stderr": 20.0, "cost_per_test": 0.106311, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "inception/mercury-3": { "accuracy": 30.0, "latency": 1470.158, "stderr": 20.0, "cost_per_test": 0.360777, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 30.0, "latency": 4340.334, "stderr": 20.0, "cost_per_test": 5.381658, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 20.0, "latency": 727.476, "stderr": 20.0, "cost_per_test": 1.084471, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": 1849.72, "stderr": 20.0, "cost_per_test": 4.143899, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 20.0, "latency": 4100.024, "stderr": 20.0, "cost_per_test": 21.45996, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 10.0, "latency": 785.382, "stderr": 10.0, "cost_per_test": 0.344122, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse-wonder": { "accuracy": 10.0, "latency": 1090.864, "stderr": 10.0, "cost_per_test": 12.604977, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 10.0, "latency": 1901.926, "stderr": 10.0, "cost_per_test": 0.565023, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 10.0, "latency": 2500.89, "stderr": 10.0, "cost_per_test": 0.275778, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 10.0, "latency": 3291.178, "stderr": 10.0, "cost_per_test": 5.61229, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 10.0, "latency": 3871.644, "stderr": 10.0, "cost_per_test": 5.123026, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 10.0, "latency": 5031.938, "stderr": 10.0, "cost_per_test": 19.376575, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 468.998, "stderr": 0.0, "cost_per_test": 0.160247, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 720.322, "stderr": 0.0, "cost_per_test": 1.427877, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 1365.906, "stderr": 0.0, "cost_per_test": 3.568526, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1405.04, "stderr": 0.0, "cost_per_test": 0.293666, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 2055.006, "stderr": 0.0, "cost_per_test": 1.20772, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 2380.55, "stderr": 0.0, "cost_per_test": 36.827844, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2848.396, "stderr": 0.0, "cost_per_test": 3.952966, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 3055.95, "stderr": 0.0, "cost_per_test": 4.810012, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "healthcare_life_sciences": { "google/gemini-4-argon": { "accuracy": 60.0, "latency": 2163.428, "stderr": 24.495, "cost_per_test": 8.843854, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 40.0, "latency": 802.956, "stderr": 24.495, "cost_per_test": 0.469585, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 40.0, "latency": 1642.936, "stderr": 24.495, "cost_per_test": 0.21495, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 40.0, "latency": 2360.19, "stderr": 24.495, "cost_per_test": 3.296002, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 40.0, "latency": 2431.536, "stderr": 24.495, "cost_per_test": 4.871642, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 40.0, "latency": 2539.434, "stderr": 18.708, "cost_per_test": 4.848495, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 40.0, "latency": 3257.984, "stderr": 24.495, "cost_per_test": 3.63425, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse-spark-open": { "accuracy": 40.0, "latency": 9853.696, "stderr": 24.495, "cost_per_test": 16.770892, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 30.0, "latency": 1125.866, "stderr": 20.0, "cost_per_test": 3.474289, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 30.0, "latency": 1457.208, "stderr": 20.0, "cost_per_test": 0.656838, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 30.0, "latency": 1854.082, "stderr": 20.0, "cost_per_test": 3.150292, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 20.0, "latency": 115.612, "stderr": 20.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 20.0, "latency": 581.14, "stderr": 20.0, "cost_per_test": 0.741568, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "inception/mercury-3": { "accuracy": 20.0, "latency": 769.502, "stderr": 20.0, "cost_per_test": 0.17794, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "meta/muse-wonder": { "accuracy": 20.0, "latency": 975.282, "stderr": 20.0, "cost_per_test": 5.625299, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 20.0, "latency": 1851.156, "stderr": 20.0, "cost_per_test": 0.526994, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "grok/grok-4.7": { "accuracy": 20.0, "latency": 2275.232, "stderr": 20.0, "cost_per_test": 11.67739, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 20.0, "latency": 2608.918, "stderr": 20.0, "cost_per_test": 3.982873, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 20.0, "latency": 4038.166, "stderr": 20.0, "cost_per_test": 28.864668, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 20.0, "latency": 5352.558, "stderr": 20.0, "cost_per_test": 19.894952, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 10.0, "latency": 528.81, "stderr": 10.0, "cost_per_test": 0.199914, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 10.0, "latency": 922.306, "stderr": 10.0, "cost_per_test": 0.09248, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 10.0, "latency": 1362.432, "stderr": 10.0, "cost_per_test": 0.267572, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 10.0, "latency": 2205.706, "stderr": 10.0, "cost_per_test": 22.288291, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 830.858, "stderr": 0.0, "cost_per_test": 1.476459, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1760.07, "stderr": 0.0, "cost_per_test": 1.267174, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2990.288, "stderr": 0.0, "cost_per_test": 3.628876, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null } }, "immigration": { "meta/muse-wonder": { "accuracy": 40.0, "latency": 622.17, "stderr": 24.495, "cost_per_test": 3.50291, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 20.0, "latency": 483.414, "stderr": 20.0, "cost_per_test": 0.526006, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 20.0, "latency": 1131.4, "stderr": 20.0, "cost_per_test": 0.37483, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 20.0, "latency": 1218.928, "stderr": 20.0, "cost_per_test": 0.468313, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 20.0, "latency": 1376.798, "stderr": 20.0, "cost_per_test": 0.79121, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": 1749.038, "stderr": 20.0, "cost_per_test": 2.461041, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 1779.824, "stderr": 20.0, "cost_per_test": 0.178109, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-wonder-full": { "accuracy": 20.0, "latency": 2604.304, "stderr": 20.0, "cost_per_test": 4.272067, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 20.0, "latency": 2807.95, "stderr": 20.0, "cost_per_test": 2.289267, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 20.0, "latency": 3787.048, "stderr": 20.0, "cost_per_test": 13.84088, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 20.0, "latency": 8923.964, "stderr": 20.0, "cost_per_test": 5.294027, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4.7": { "accuracy": 10.0, "latency": 2388.162, "stderr": 10.0, "cost_per_test": 9.158315, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 10.0, "latency": 3151.272, "stderr": 10.0, "cost_per_test": 18.52264, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 114.494, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 362.142, "stderr": 0.0, "cost_per_test": 0.851971, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 485.528, "stderr": 0.0, "cost_per_test": 0.15155, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 659.556, "stderr": 0.0, "cost_per_test": 0.074179, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 663.922, "stderr": 0.0, "cost_per_test": 0.2425, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 899.282, "stderr": 0.0, "cost_per_test": 0.069274, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 1107.642, "stderr": 0.0, "cost_per_test": 2.608087, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1151.354, "stderr": 0.0, "cost_per_test": 0.130511, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 1343.676, "stderr": 0.0, "cost_per_test": 9.272635, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1584.174, "stderr": 0.0, "cost_per_test": 11.992363, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 1745.378, "stderr": 0.0, "cost_per_test": 1.539912, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 1942.628, "stderr": 0.0, "cost_per_test": 2.10625, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 2070.532, "stderr": 0.0, "cost_per_test": 2.772501, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2118.592, "stderr": 0.0, "cost_per_test": 2.855998, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "insurance": { "meta/muse-wonder": { "accuracy": 30.0, "latency": 921.1, "stderr": 20.0, "cost_per_test": 7.517888, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 30.0, "latency": 2466.874, "stderr": 20.0, "cost_per_test": 4.928353, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 20.0, "latency": 1446.246, "stderr": 20.0, "cost_per_test": 6.035978, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 10.0, "latency": 1686.474, "stderr": 10.0, "cost_per_test": 14.040593, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 141.572, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 220.402, "stderr": 0.0, "cost_per_test": 0.135413, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 351.262, "stderr": 0.0, "cost_per_test": 1.138776, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 533.98, "stderr": 0.0, "cost_per_test": 0.667337, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 648.634, "stderr": 0.0, "cost_per_test": 0.231492, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 854.286, "stderr": 0.0, "cost_per_test": 0.088321, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 980.822, "stderr": 0.0, "cost_per_test": 0.194079, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 0.0, "latency": 1028.702, "stderr": 0.0, "cost_per_test": 0.343356, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 0.0, "latency": 1109.874, "stderr": 0.0, "cost_per_test": 0.476309, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 1280.274, "stderr": 0.0, "cost_per_test": 3.594349, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 1284.864, "stderr": 0.0, "cost_per_test": 0.060043, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1644.074, "stderr": 0.0, "cost_per_test": 0.168232, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 1679.992, "stderr": 0.0, "cost_per_test": 2.315951, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 1803.898, "stderr": 0.0, "cost_per_test": 2.905753, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1813.96, "stderr": 0.0, "cost_per_test": 0.879072, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2009.804, "stderr": 0.0, "cost_per_test": 2.136981, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2030.584, "stderr": 0.0, "cost_per_test": 9.139267, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 2051.788, "stderr": 0.0, "cost_per_test": 2.912115, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2167.052, "stderr": 0.0, "cost_per_test": 3.276103, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 0.0, "latency": 2472.566, "stderr": 0.0, "cost_per_test": 2.316687, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3072.686, "stderr": 0.0, "cost_per_test": 16.375474, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 3563.796, "stderr": 0.0, "cost_per_test": 15.744184, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 0.0, "latency": 16077.45, "stderr": 0.0, "cost_per_test": 17.909109, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null } }, "intellectual_property": { "meta/muse-spark-open": { "accuracy": 50.0, "latency": 16950.634, "stderr": 22.361, "cost_per_test": 14.304537, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 40.0, "latency": 1994.312, "stderr": 18.708, "cost_per_test": 10.686247, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 40.0, "latency": 2441.248, "stderr": 18.708, "cost_per_test": 3.435719, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 40.0, "latency": 2626.582, "stderr": 18.708, "cost_per_test": 3.255216, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 30.0, "latency": 978.73, "stderr": 20.0, "cost_per_test": 3.241614, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3": { "accuracy": 30.0, "latency": 1690.696, "stderr": 20.0, "cost_per_test": 3.514176, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 30.0, "latency": 1923.054, "stderr": 20.0, "cost_per_test": 1.123387, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "meta/muse-wonder-full": { "accuracy": 30.0, "latency": 2474.152, "stderr": 20.0, "cost_per_test": 5.894615, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 30.0, "latency": 4362.09, "stderr": 20.0, "cost_per_test": 17.055475, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 20.0, "latency": 815.826, "stderr": 20.0, "cost_per_test": 0.359106, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse-wonder": { "accuracy": 20.0, "latency": 818.0, "stderr": 20.0, "cost_per_test": 7.383931, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 20.0, "latency": 1122.132, "stderr": 20.0, "cost_per_test": 0.088739, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 20.0, "latency": 1452.402, "stderr": 20.0, "cost_per_test": 0.496673, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "grok/grok-4.7": { "accuracy": 20.0, "latency": 2175.528, "stderr": 12.247, "cost_per_test": 10.417022, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inception/mercury-3": { "accuracy": 10.0, "latency": 546.538, "stderr": 10.0, "cost_per_test": 0.135627, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 10.0, "latency": 655.82, "stderr": 10.0, "cost_per_test": 0.858728, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 10.0, "latency": 1253.686, "stderr": 10.0, "cost_per_test": 0.259407, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 10.0, "latency": 1311.01, "stderr": 10.0, "cost_per_test": 0.601227, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 10.0, "latency": 1761.748, "stderr": 10.0, "cost_per_test": 0.205832, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-astra": { "accuracy": 10.0, "latency": 1837.104, "stderr": 10.0, "cost_per_test": 19.556639, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 10.0, "latency": 2049.792, "stderr": 10.0, "cost_per_test": 2.395422, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 10.0, "latency": 2882.972, "stderr": 10.0, "cost_per_test": 4.592565, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 10.0, "latency": 3302.178, "stderr": 10.0, "cost_per_test": 5.422039, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 10.0, "latency": 3385.322, "stderr": 10.0, "cost_per_test": 20.701364, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 109.824, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 441.472, "stderr": 0.0, "cost_per_test": 0.163547, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 702.04, "stderr": 0.0, "cost_per_test": 1.137063, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "international_trade_sanctions": { "thinkingmachines/inkling": { "accuracy": 30.0, "latency": 581.252, "stderr": 20.0, "cost_per_test": 1.18382, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse-wonder": { "accuracy": 30.0, "latency": 902.572, "stderr": 20.0, "cost_per_test": 3.602648, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "reflection/Beam-1.0": { "accuracy": 20.0, "latency": 111.886, "stderr": 20.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 20.0, "latency": 1212.502, "stderr": 20.0, "cost_per_test": 3.921448, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": 1643.282, "stderr": 20.0, "cost_per_test": 2.37339, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 20.0, "latency": 2063.232, "stderr": 12.247, "cost_per_test": 4.26355, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "kimi/kimi-k3": { "accuracy": 20.0, "latency": 2164.224, "stderr": 20.0, "cost_per_test": 2.611892, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 20.0, "latency": 2588.454, "stderr": 20.0, "cost_per_test": 2.262196, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 20.0, "latency": 6327.422, "stderr": 20.0, "cost_per_test": 3.46041, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 20.0, "latency": 10378.424, "stderr": 12.247, "cost_per_test": 4.714199, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "inception/mercury-3": { "accuracy": 10.0, "latency": 583.14, "stderr": 10.0, "cost_per_test": 0.086833, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 10.0, "latency": 1014.756, "stderr": 10.0, "cost_per_test": 0.06912, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-4-argon": { "accuracy": 10.0, "latency": 1409.552, "stderr": 10.0, "cost_per_test": 7.007018, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 10.0, "latency": 2203.268, "stderr": 10.0, "cost_per_test": 9.257126, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 10.0, "latency": 2343.334, "stderr": 10.0, "cost_per_test": 0.787466, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 515.1, "stderr": 0.0, "cost_per_test": 0.174373, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 639.65, "stderr": 0.0, "cost_per_test": 0.64046, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 774.376, "stderr": 0.0, "cost_per_test": 0.257926, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 982.246, "stderr": 0.0, "cost_per_test": 0.115275, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 0.0, "latency": 1463.816, "stderr": 0.0, "cost_per_test": 0.612386, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 0.0, "latency": 1530.654, "stderr": 0.0, "cost_per_test": 0.355021, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1668.724, "stderr": 0.0, "cost_per_test": 12.013329, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1736.24, "stderr": 0.0, "cost_per_test": 0.192448, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2166.526, "stderr": 0.0, "cost_per_test": 1.991433, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2319.978, "stderr": 0.0, "cost_per_test": 3.414516, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3447.774, "stderr": 0.0, "cost_per_test": 20.677557, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 4004.068, "stderr": 0.0, "cost_per_test": 16.269947, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "litigation_dispute_resolution": { "grok/grok-4.7": { "accuracy": 40.0, "latency": 2256.59, "stderr": 24.495, "cost_per_test": 11.087814, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 30.0, "latency": 1530.146, "stderr": 20.0, "cost_per_test": 0.914139, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3": { "accuracy": 30.0, "latency": 1897.458, "stderr": 20.0, "cost_per_test": 3.967749, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 30.0, "latency": 3012.004, "stderr": 20.0, "cost_per_test": 3.511846, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse-spark-open": { "accuracy": 30.0, "latency": 15708.348, "stderr": 12.247, "cost_per_test": 14.973506, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 20.0, "latency": 624.304, "stderr": 20.0, "cost_per_test": 0.817563, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 20.0, "latency": 817.96, "stderr": 20.0, "cost_per_test": 0.371947, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 20.0, "latency": 924.466, "stderr": 20.0, "cost_per_test": 0.089255, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 20.0, "latency": 1129.724, "stderr": 12.247, "cost_per_test": 0.577367, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 20.0, "latency": 2669.568, "stderr": 20.0, "cost_per_test": 4.340162, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 20.0, "latency": 5728.15, "stderr": 20.0, "cost_per_test": 4.311466, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse-wonder": { "accuracy": 10.0, "latency": 850.952, "stderr": 10.0, "cost_per_test": 10.904622, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 10.0, "latency": 1218.222, "stderr": 10.0, "cost_per_test": 4.594796, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 10.0, "latency": 1470.038, "stderr": 10.0, "cost_per_test": 0.48408, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-astra": { "accuracy": 10.0, "latency": 1809.34, "stderr": 10.0, "cost_per_test": 33.102, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 10.0, "latency": 1810.548, "stderr": 10.0, "cost_per_test": 10.28651, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 10.0, "latency": 2566.032, "stderr": 10.0, "cost_per_test": 4.235701, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 121.658, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 548.538, "stderr": 0.0, "cost_per_test": 0.200138, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 718.642, "stderr": 0.0, "cost_per_test": 1.209207, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 1189.146, "stderr": 0.0, "cost_per_test": 0.276764, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1784.046, "stderr": 0.0, "cost_per_test": 0.217599, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 2139.15, "stderr": 0.0, "cost_per_test": 0.869002, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 0.0, "latency": 2486.446, "stderr": 0.0, "cost_per_test": 5.721406, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2621.26, "stderr": 0.0, "cost_per_test": 4.966411, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 4130.842, "stderr": 0.0, "cost_per_test": 21.148726, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 4470.472, "stderr": 0.0, "cost_per_test": 13.912957, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "real_estate": { "meta/muse-wonder-full": { "accuracy": 40.0, "latency": 2871.286, "stderr": 24.495, "cost_per_test": 6.578616, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 30.0, "latency": 790.378, "stderr": 20.0, "cost_per_test": 5.727136, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 30.0, "latency": 2034.836, "stderr": 20.0, "cost_per_test": 12.048195, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 30.0, "latency": 2897.188, "stderr": 20.0, "cost_per_test": 5.263349, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 30.0, "latency": 2926.518, "stderr": 20.0, "cost_per_test": 5.105426, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 20.0, "latency": 733.85, "stderr": 20.0, "cost_per_test": 0.442482, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 20.0, "latency": 1191.344, "stderr": 20.0, "cost_per_test": 0.101049, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 20.0, "latency": 1748.814, "stderr": 20.0, "cost_per_test": 0.266735, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 20.0, "latency": 2713.792, "stderr": 20.0, "cost_per_test": 4.688049, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 20.0, "latency": 5189.844, "stderr": 20.0, "cost_per_test": 16.877102, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 10.0, "latency": 106.536, "stderr": 10.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 10.0, "latency": 726.292, "stderr": 10.0, "cost_per_test": 1.119961, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 10.0, "latency": 1120.304, "stderr": 10.0, "cost_per_test": 4.5574, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 10.0, "latency": 1407.424, "stderr": 10.0, "cost_per_test": 0.669758, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-4-argon": { "accuracy": 10.0, "latency": 2112.972, "stderr": 10.0, "cost_per_test": 16.654201, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 10.0, "latency": 3861.442, "stderr": 10.0, "cost_per_test": 21.779478, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 10.0, "latency": 19874.44, "stderr": 10.0, "cost_per_test": 25.105877, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 437.088, "stderr": 0.0, "cost_per_test": 1.394793, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 452.006, "stderr": 0.0, "cost_per_test": 0.140612, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 946.294, "stderr": 0.0, "cost_per_test": 0.251521, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 0.0, "latency": 1428.066, "stderr": 0.0, "cost_per_test": 0.471568, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1435.644, "stderr": 0.0, "cost_per_test": 0.466666, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1577.124, "stderr": 0.0, "cost_per_test": 1.123838, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 1838.822, "stderr": 0.0, "cost_per_test": 4.97377, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 1943.306, "stderr": 0.0, "cost_per_test": 4.477501, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 2089.27, "stderr": 0.0, "cost_per_test": 27.678718, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 0.0, "latency": 2466.112, "stderr": 0.0, "cost_per_test": 3.079066, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null } }, "structured_finance_securitization": { "google/gemini-4-argon": { "accuracy": 30.0, "latency": 1931.426, "stderr": 20.0, "cost_per_test": 13.409945, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 30.0, "latency": 4095.526, "stderr": 20.0, "cost_per_test": 19.757045, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 20.0, "latency": 786.548, "stderr": 20.0, "cost_per_test": 0.303097, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse-wonder": { "accuracy": 20.0, "latency": 969.446, "stderr": 20.0, "cost_per_test": 7.587681, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 20.0, "latency": 1073.882, "stderr": 20.0, "cost_per_test": 0.092304, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 20.0, "latency": 1092.26, "stderr": 20.0, "cost_per_test": 0.430794, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 20.0, "latency": 1214.128, "stderr": 20.0, "cost_per_test": 0.5812, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": 1620.42, "stderr": 20.0, "cost_per_test": 3.680402, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.7": { "accuracy": 20.0, "latency": 2069.346, "stderr": 20.0, "cost_per_test": 10.094563, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 20.0, "latency": 2426.824, "stderr": 20.0, "cost_per_test": 6.8176, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 20.0, "latency": 2433.226, "stderr": 20.0, "cost_per_test": 4.677121, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 20.0, "latency": 15069.208, "stderr": 20.0, "cost_per_test": 6.985702, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 10.0, "latency": 1876.594, "stderr": 10.0, "cost_per_test": 0.212322, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 10.0, "latency": 2592.83, "stderr": 10.0, "cost_per_test": 2.813511, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 10.0, "latency": 3832.726, "stderr": 10.0, "cost_per_test": 13.720065, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 106.582, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 273.318, "stderr": 0.0, "cost_per_test": 1.246746, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 499.992, "stderr": 0.0, "cost_per_test": 0.181086, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 517.66, "stderr": 0.0, "cost_per_test": 0.690061, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 1084.012, "stderr": 0.0, "cost_per_test": 3.302349, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 1222.948, "stderr": 0.0, "cost_per_test": 0.179295, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1362.744, "stderr": 0.0, "cost_per_test": 0.326133, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1527.02, "stderr": 0.0, "cost_per_test": 0.84184, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 2092.05, "stderr": 0.0, "cost_per_test": 24.947157, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 2283.332, "stderr": 0.0, "cost_per_test": 5.207436, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2638.612, "stderr": 0.0, "cost_per_test": 3.667821, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 3025.816, "stderr": 0.0, "cost_per_test": 5.220471, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "tax": { "google/gemini-4-argon": { "accuracy": 20.0, "latency": 1632.154, "stderr": 20.0, "cost_per_test": 10.875562, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 20.0, "latency": 4033.49, "stderr": 20.0, "cost_per_test": 6.118001, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 20.0, "latency": 4537.142, "stderr": 20.0, "cost_per_test": 4.442674, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse-spark-open": { "accuracy": 20.0, "latency": 11961.888, "stderr": 20.0, "cost_per_test": 10.001683, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 10.0, "latency": 774.622, "stderr": 10.0, "cost_per_test": 1.050716, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 10.0, "latency": 1144.05, "stderr": 10.0, "cost_per_test": 0.501765, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 10.0, "latency": 1262.568, "stderr": 10.0, "cost_per_test": 4.824852, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 10.0, "latency": 1330.648, "stderr": 10.0, "cost_per_test": 0.096872, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 10.0, "latency": 1437.404, "stderr": 10.0, "cost_per_test": 0.703428, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 10.0, "latency": 2086.784, "stderr": 10.0, "cost_per_test": 0.247354, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 10.0, "latency": 2328.808, "stderr": 10.0, "cost_per_test": 5.63308, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 10.0, "latency": 2859.728, "stderr": 10.0, "cost_per_test": 5.054813, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 110.238, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 430.432, "stderr": 0.0, "cost_per_test": 0.915144, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 525.194, "stderr": 0.0, "cost_per_test": 0.205472, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse-wonder": { "accuracy": 0.0, "latency": 873.104, "stderr": 0.0, "cost_per_test": 5.425917, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 911.938, "stderr": 0.0, "cost_per_test": 0.160866, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1376.274, "stderr": 0.0, "cost_per_test": 0.329521, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 0.0, "latency": 1483.114, "stderr": 0.0, "cost_per_test": 0.548613, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 2204.202, "stderr": 0.0, "cost_per_test": 39.15058, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 0.0, "latency": 2475.476, "stderr": 0.0, "cost_per_test": 5.723511, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2523.416, "stderr": 0.0, "cost_per_test": 3.793071, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 2535.448, "stderr": 0.0, "cost_per_test": 11.489974, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2757.38, "stderr": 0.0, "cost_per_test": 4.69923, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 2999.882, "stderr": 0.0, "cost_per_test": 1.092562, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3778.528, "stderr": 0.0, "cost_per_test": 17.274333, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 4877.428, "stderr": 0.0, "cost_per_test": 16.089025, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "trusts_estates_private_client": { "meta/muse-wonder-full": { "accuracy": 40.0, "latency": 2998.392, "stderr": 18.708, "cost_per_test": 4.090553, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 30.0, "latency": 1226.72, "stderr": 20.0, "cost_per_test": 4.184536, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-4-argon": { "accuracy": 30.0, "latency": 1572.052, "stderr": 20.0, "cost_per_test": 16.183565, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 30.0, "latency": 1947.846, "stderr": 20.0, "cost_per_test": 0.23816, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "grok/grok-4.7": { "accuracy": 30.0, "latency": 2252.568, "stderr": 12.247, "cost_per_test": 11.951101, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 30.0, "latency": 3115.438, "stderr": 20.0, "cost_per_test": 3.818917, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse-spark-open": { "accuracy": 30.0, "latency": 13246.212, "stderr": 20.0, "cost_per_test": 15.744338, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse-wonder": { "accuracy": 20.0, "latency": 920.036, "stderr": 20.0, "cost_per_test": 7.713584, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 20.0, "latency": 1358.138, "stderr": 20.0, "cost_per_test": 0.610376, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 20.0, "latency": 1652.888, "stderr": 20.0, "cost_per_test": 0.088474, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 20.0, "latency": 2075.326, "stderr": 12.247, "cost_per_test": 5.517077, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-5.3": { "accuracy": 20.0, "latency": 2222.098, "stderr": 12.247, "cost_per_test": 5.43764, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 20.0, "latency": 3542.812, "stderr": 20.0, "cost_per_test": 21.781833, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 20.0, "latency": 4217.882, "stderr": 20.0, "cost_per_test": 15.106108, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 10.0, "latency": 303.938, "stderr": 10.0, "cost_per_test": 0.186361, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 10.0, "latency": 420.692, "stderr": 10.0, "cost_per_test": 1.392685, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "tencent/hy4-preview": { "accuracy": 10.0, "latency": 1410.15, "stderr": 10.0, "cost_per_test": 0.915369, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-astra": { "accuracy": 10.0, "latency": 1899.114, "stderr": 10.0, "cost_per_test": 23.695343, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 10.0, "latency": 1956.23, "stderr": 10.0, "cost_per_test": 2.971612, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 10.0, "latency": 2335.938, "stderr": 10.0, "cost_per_test": 3.927228, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 115.176, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 653.918, "stderr": 0.0, "cost_per_test": 0.89094, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 762.432, "stderr": 0.0, "cost_per_test": 0.369984, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1080.394, "stderr": 0.0, "cost_per_test": 0.243579, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 1327.644, "stderr": 0.0, "cost_per_test": 0.274585, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 0.0, "latency": 1486.298, "stderr": 0.0, "cost_per_test": 0.4795, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 2515.052, "stderr": 0.0, "cost_per_test": 3.805345, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "white_collar_defense_investigations": { "meta/muse-wonder": { "accuracy": 20.0, "latency": 1201.638, "stderr": 20.0, "cost_per_test": 6.121717, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "reflection/Beam-1.0": { "accuracy": 0.0, "latency": 109.104, "stderr": 0.0, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling": { "accuracy": 0.0, "latency": 374.084, "stderr": 0.0, "cost_per_test": 1.451903, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 0.0, "latency": 433.434, "stderr": 0.0, "cost_per_test": 0.180201, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 0.0, "latency": 477.316, "stderr": 0.0, "cost_per_test": 0.622985, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 0.0, "latency": 694.844, "stderr": 0.0, "cost_per_test": 0.286703, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-3": { "accuracy": 0.0, "latency": 714.676, "stderr": 0.0, "cost_per_test": 0.134735, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 0.0, "latency": 917.02, "stderr": 0.0, "cost_per_test": 0.07138, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 0.0, "latency": 1076.182, "stderr": 0.0, "cost_per_test": 3.121405, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 0.0, "latency": 1357.85, "stderr": 0.0, "cost_per_test": 0.463122, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 0.0, "latency": 1431.654, "stderr": 0.0, "cost_per_test": 0.688859, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-luna": { "accuracy": 0.0, "latency": 1480.506, "stderr": 0.0, "cost_per_test": 0.268224, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 0.0, "latency": 1697.31, "stderr": 0.0, "cost_per_test": 0.869928, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 0.0, "latency": 1712.104, "stderr": 0.0, "cost_per_test": 0.16563, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-astra": { "accuracy": 0.0, "latency": 1732.526, "stderr": 0.0, "cost_per_test": 17.857837, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-4-argon": { "accuracy": 0.0, "latency": 1880.166, "stderr": 0.0, "cost_per_test": 10.940583, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 0.0, "latency": 1976.236, "stderr": 0.0, "cost_per_test": 9.182441, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 0.0, "latency": 2123.426, "stderr": 0.0, "cost_per_test": 3.053435, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3": { "accuracy": 0.0, "latency": 2147.146, "stderr": 0.0, "cost_per_test": 4.334048, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 0.0, "latency": 2455.672, "stderr": 0.0, "cost_per_test": 3.663112, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 0.0, "latency": 2595.43, "stderr": 0.0, "cost_per_test": 5.453971, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 0.0, "latency": 2749.246, "stderr": 0.0, "cost_per_test": 2.850712, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k3": { "accuracy": 0.0, "latency": 2825.84, "stderr": 0.0, "cost_per_test": 5.839684, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 0.0, "latency": 3043.182, "stderr": 0.0, "cost_per_test": 5.01937, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 0.0, "latency": 3518.708, "stderr": 0.0, "cost_per_test": 24.223778, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 0.0, "latency": 4937.706, "stderr": 0.0, "cost_per_test": 17.206836, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 0.0, "latency": 11793.734, "stderr": 0.0, "cost_per_test": 8.882501, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null } }, "criteria_pass_rate": { "google/gemini-4-argon": { "accuracy": 96.3, "latency": 1755.397, "stderr": 0.227, "cost_per_test": 10.54836, "token_totals": { "input_tokens": 812145091, "output_tokens": 18424207, "reasoning_tokens": 5211457, "cache_read_tokens": 733823568, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 96.126, "latency": 2649.375, "stderr": 0.232, "cost_per_test": 5.413598, "token_totals": { "input_tokens": 1349672470, "output_tokens": 13499459, "reasoning_tokens": 2307478, "cache_read_tokens": 1308495584, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 96.01, "latency": 906.14, "stderr": 0.236, "cost_per_test": 7.081987, "token_totals": { "input_tokens": 1414833931, "output_tokens": 12111113, "reasoning_tokens": 1875186, "cache_read_tokens": 1374979944, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 93.848, "latency": 2950.56, "stderr": 0.289, "cost_per_test": 3.358663, "token_totals": { "input_tokens": 934205246, "output_tokens": 14880894, "reasoning_tokens": 6584092, "cache_read_tokens": 888375040, "cache_write_tokens": null }, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 93.717, "latency": 1911.01, "stderr": 0.292, "cost_per_test": 3.987774, "token_totals": { "input_tokens": 1376154444, "output_tokens": 16202326, "reasoning_tokens": 7676761, "cache_read_tokens": 1332783858, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.7": { "accuracy": 93.652, "latency": 2148.428, "stderr": 0.294, "cost_per_test": 10.746259, "token_totals": { "input_tokens": 1088222870, "output_tokens": 20549074, "reasoning_tokens": 8836231, "cache_read_tokens": 1043005952, "cache_write_tokens": 0 }, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 93.485, "latency": 2556.302, "stderr": 0.297, "cost_per_test": 4.144144, "token_totals": { "input_tokens": 887032678, "output_tokens": 10027338, "reasoning_tokens": 3826649, "cache_read_tokens": 857115099, "cache_write_tokens": 29371546 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 93.006, "latency": 3483.562, "stderr": 0.307, "cost_per_test": 4.181392, "token_totals": { "input_tokens": 813025818, "output_tokens": 12794152, "reasoning_tokens": null, "cache_read_tokens": 459528168, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 92.876, "latency": 1861.848, "stderr": 0.31, "cost_per_test": 0.216614, "token_totals": { "input_tokens": 631569981, "output_tokens": 10405619, "reasoning_tokens": 4622671, "cache_read_tokens": 604899072, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 92.803, "latency": 1342.563, "stderr": 0.311, "cost_per_test": 0.085646, "token_totals": { "input_tokens": 958015494, "output_tokens": 12423312, "reasoning_tokens": 6316080, "cache_read_tokens": 925243136, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 92.665, "latency": 1328.108, "stderr": 0.314, "cost_per_test": 0.612596, "token_totals": { "input_tokens": 5524481146, "output_tokens": 25881985, "reasoning_tokens": 10520362, "cache_read_tokens": 5492827392, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 92.636, "latency": 1161.715, "stderr": 0.315, "cost_per_test": 3.656207, "token_totals": { "input_tokens": 1288268290, "output_tokens": 16504243, "reasoning_tokens": 6011153, "cache_read_tokens": 1232055515, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 91.867, "latency": 655.413, "stderr": 0.329, "cost_per_test": 0.802111, "token_totals": { "input_tokens": 746916941, "output_tokens": 9384470, "reasoning_tokens": 4362757, "cache_read_tokens": 726363264, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 91.708, "latency": 1953.362, "stderr": 0.332, "cost_per_test": 25.460295, "token_totals": { "input_tokens": 1310790488, "output_tokens": 10401199, "reasoning_tokens": 3452073, "cache_read_tokens": 1276652993, "cache_write_tokens": 34120712 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 91.563, "latency": 744.843, "stderr": 0.335, "cost_per_test": 0.323243, "token_totals": { "input_tokens": 1006432418, "output_tokens": 9841376, "reasoning_tokens": 3565890, "cache_read_tokens": 975803052, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 91.483, "latency": 2425.625, "stderr": 0.336, "cost_per_test": 3.536302, "token_totals": { "input_tokens": 1227717408, "output_tokens": 11158751, "reasoning_tokens": 3901277, "cache_read_tokens": 1191414163, "cache_write_tokens": 36281197 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 91.077, "latency": 1497.601, "stderr": 0.343, "cost_per_test": 0.472848, "token_totals": { "input_tokens": 2730849267, "output_tokens": 20117381, "reasoning_tokens": 6681765, "cache_read_tokens": 2703378118, "cache_write_tokens": 0 }, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "meta/muse-spark-open": { "accuracy": 90.112, "latency": 14355.684, "stderr": 0.36, "cost_per_test": 13.6316, "token_totals": { "input_tokens": 2657733432, "output_tokens": 13857658, "reasoning_tokens": 6185749, "cache_read_tokens": 1586608896, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 90.104, "latency": 2620.795, "stderr": 0.36, "cost_per_test": 4.264376, "token_totals": { "input_tokens": 4904885383, "output_tokens": 69221549, "reasoning_tokens": 47929104, "cache_read_tokens": 4716242036, "cache_write_tokens": 188617435 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 89.793, "latency": 3507.428, "stderr": 0.365, "cost_per_test": 21.37949, "token_totals": { "input_tokens": 3749613119, "output_tokens": 44884808, "reasoning_tokens": 27684473, "cache_read_tokens": 3547556427, "cache_write_tokens": 202035592 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "inception/mercury-3": { "accuracy": 89.611, "latency": 886.884, "stderr": 0.368, "cost_per_test": 0.188085, "token_totals": { "input_tokens": 659070144, "output_tokens": 6092886, "reasoning_tokens": 5836559, "cache_read_tokens": 505214083, "cache_write_tokens": null }, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "tencent/hy4-preview": { "accuracy": 89.604, "latency": 1989.335, "stderr": 0.368, "cost_per_test": 0.98378, "token_totals": { "input_tokens": 1373541600, "output_tokens": 12484050, "reasoning_tokens": 4013313, "cache_read_tokens": 1314797886, "cache_write_tokens": 0 }, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "reflection/Beam-1.0": { "accuracy": 89.06, "latency": 116.883, "stderr": 0.376, "cost_per_test": null, "token_totals": { "input_tokens": 540211238, "output_tokens": 4325040, "reasoning_tokens": 1196691, "cache_read_tokens": 509200704, "cache_write_tokens": null }, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "openai/gpt-6-luna": { "accuracy": 88.298, "latency": 1305.488, "stderr": 0.387, "cost_per_test": 0.298018, "token_totals": { "input_tokens": 1322468906, "output_tokens": 12364131, "reasoning_tokens": 6724988, "cache_read_tokens": 1277646907, "cache_write_tokens": 44806375 }, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 85.947, "latency": 4397.953, "stderr": 0.419, "cost_per_test": 16.612583, "token_totals": { "input_tokens": 7599893900, "output_tokens": 71504693, "reasoning_tokens": 47513756, "cache_read_tokens": 7328811056, "cache_write_tokens": 271040714 }, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 81.493, "latency": 455.265, "stderr": 0.468, "cost_per_test": 0.17869, "token_totals": { "input_tokens": 200497219, "output_tokens": 5802466, "reasoning_tokens": null, "cache_read_tokens": 190288768, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 72.766, "latency": 529.644, "stderr": 0.536, "cost_per_test": 1.252023, "token_totals": { "input_tokens": 538710546, "output_tokens": 7813620, "reasoning_tokens": null, "cache_read_tokens": 513145856, "cache_write_tokens": null }, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__antitrust_competition": { "google/gemini-4-argon": { "accuracy": 97.645, "latency": 1611.726, "stderr": 0.914, "cost_per_test": 15.535189, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 96.92, "latency": 2044.898, "stderr": 1.042, "cost_per_test": 6.005504, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-spark-open": { "accuracy": 96.377, "latency": 12012.07, "stderr": 1.127, "cost_per_test": 3.976357, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse-wonder": { "accuracy": 95.652, "latency": 1023.55, "stderr": 1.23, "cost_per_test": 7.544454, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 93.659, "latency": 1690.59, "stderr": 1.47, "cost_per_test": 0.192669, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 93.478, "latency": 1264.71, "stderr": 1.489, "cost_per_test": 0.534289, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 93.478, "latency": 1538.864, "stderr": 1.489, "cost_per_test": 2.533451, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.7": { "accuracy": 93.478, "latency": 1997.668, "stderr": 1.489, "cost_per_test": 9.438476, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 93.297, "latency": 5773.048, "stderr": 1.508, "cost_per_test": 3.909019, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 93.116, "latency": 2346.238, "stderr": 1.527, "cost_per_test": 3.732132, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 92.935, "latency": 890.898, "stderr": 1.545, "cost_per_test": 3.871498, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 92.935, "latency": 2261.178, "stderr": 1.545, "cost_per_test": 2.589032, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 92.572, "latency": 691.274, "stderr": 1.581, "cost_per_test": 0.266224, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 92.572, "latency": 2364.752, "stderr": 1.581, "cost_per_test": 0.077418, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 92.029, "latency": 1602.082, "stderr": 1.633, "cost_per_test": 0.447689, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-astra": { "accuracy": 91.304, "latency": 1978.794, "stderr": 1.699, "cost_per_test": 28.281218, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-3": { "accuracy": 91.123, "latency": 1001.484, "stderr": 1.715, "cost_per_test": 0.196264, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "tencent/hy4-preview": { "accuracy": 90.942, "latency": 1982.912, "stderr": 1.731, "cost_per_test": 0.789632, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 90.761, "latency": 2421.532, "stderr": 1.746, "cost_per_test": 2.7654, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 90.399, "latency": 654.154, "stderr": 1.777, "cost_per_test": 0.759752, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 90.217, "latency": 4001.192, "stderr": 1.791, "cost_per_test": 17.847147, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 88.949, "latency": 104.63, "stderr": 1.891, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 88.406, "latency": 3063.884, "stderr": 1.931, "cost_per_test": 23.046646, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 86.232, "latency": 2274.678, "stderr": 2.078, "cost_per_test": 3.460397, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 84.239, "latency": 694.13, "stderr": 2.197, "cost_per_test": 1.357335, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-luna": { "accuracy": 84.239, "latency": 1062.146, "stderr": 2.197, "cost_per_test": 0.226157, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 81.522, "latency": 471.488, "stderr": 2.34, "cost_per_test": 0.170509, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__arbitration_international_dispute_resolution": { "xiaomi/mimo-v2.6-flash": { "accuracy": 96.953, "latency": 1107.096, "stderr": 1.031, "cost_per_test": 0.098258, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-spark-open": { "accuracy": 96.953, "latency": 16647.192, "stderr": 1.031, "cost_per_test": 17.977879, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 96.595, "latency": 2620.15, "stderr": 1.088, "cost_per_test": 5.114031, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 96.237, "latency": 626.442, "stderr": 1.141, "cost_per_test": 0.933152, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-4-argon": { "accuracy": 96.057, "latency": 2222.162, "stderr": 1.167, "cost_per_test": 12.524633, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 95.878, "latency": 842.758, "stderr": 1.192, "cost_per_test": 6.853377, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 95.52, "latency": 1735.19, "stderr": 1.241, "cost_per_test": 0.22831, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 95.52, "latency": 1998.852, "stderr": 1.241, "cost_per_test": 5.120208, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 95.341, "latency": 7424.862, "stderr": 1.264, "cost_per_test": 4.415408, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 95.161, "latency": 3255.392, "stderr": 1.287, "cost_per_test": 4.767735, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.7": { "accuracy": 93.907, "latency": 1874.266, "stderr": 1.435, "cost_per_test": 12.641362, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 93.907, "latency": 2832.902, "stderr": 1.435, "cost_per_test": 5.231339, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 93.369, "latency": 858.63, "stderr": 1.492, "cost_per_test": 0.546196, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 92.832, "latency": 5100.89, "stderr": 1.547, "cost_per_test": 16.01159, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "inception/mercury-3": { "accuracy": 92.115, "latency": 826.742, "stderr": 1.616, "cost_per_test": 0.220421, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 91.756, "latency": 1260.926, "stderr": 1.65, "cost_per_test": 4.266851, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 91.756, "latency": 1273.796, "stderr": 1.65, "cost_per_test": 0.66721, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 91.577, "latency": 3841.334, "stderr": 1.666, "cost_per_test": 22.280508, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 91.219, "latency": 2052.194, "stderr": 1.697, "cost_per_test": 37.28026, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 91.039, "latency": 2687.456, "stderr": 1.713, "cost_per_test": 5.097847, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 90.681, "latency": 3010.736, "stderr": 1.743, "cost_per_test": 5.265369, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 90.502, "latency": 113.746, "stderr": 1.758, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "tencent/hy4-preview": { "accuracy": 90.323, "latency": 3592.504, "stderr": 1.773, "cost_per_test": 1.100251, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-luna": { "accuracy": 90.143, "latency": 1371.914, "stderr": 1.788, "cost_per_test": 0.423667, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 88.351, "latency": 1490.442, "stderr": 1.924, "cost_per_test": 0.572096, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 75.269, "latency": 604.136, "stderr": 2.588, "cost_per_test": 0.213826, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 48.208, "latency": 692.844, "stderr": 2.997, "cost_per_test": 0.801309, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__banking_finance": { "meta/muse-wonder-full": { "accuracy": 97.575, "latency": 3124.788, "stderr": 0.941, "cost_per_test": 5.41453, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 97.388, "latency": 1393.73, "stderr": 0.976, "cost_per_test": 8.507105, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 96.455, "latency": 936.4, "stderr": 1.132, "cost_per_test": 6.016663, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 95.336, "latency": 2121.904, "stderr": 1.291, "cost_per_test": 10.337825, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 94.03, "latency": 1802.172, "stderr": 1.45, "cost_per_test": 0.071494, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 93.657, "latency": 751.412, "stderr": 1.492, "cost_per_test": 0.76189, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 93.657, "latency": 1130.544, "stderr": 1.492, "cost_per_test": 2.511915, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 93.657, "latency": 1557.112, "stderr": 1.492, "cost_per_test": 0.363031, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 93.657, "latency": 2841.088, "stderr": 1.492, "cost_per_test": 3.059804, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 93.47, "latency": 706.31, "stderr": 1.512, "cost_per_test": 0.232024, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 93.284, "latency": 2181.91, "stderr": 1.532, "cost_per_test": 3.430828, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 93.097, "latency": 1845.728, "stderr": 1.551, "cost_per_test": 26.702694, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 93.097, "latency": 3454.478, "stderr": 1.551, "cost_per_test": 16.890636, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 92.724, "latency": 1666.362, "stderr": 1.59, "cost_per_test": 3.316185, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 92.724, "latency": 1747.954, "stderr": 1.59, "cost_per_test": 0.183967, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 92.724, "latency": 2357.686, "stderr": 1.59, "cost_per_test": 3.100539, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 91.231, "latency": 2593.164, "stderr": 1.731, "cost_per_test": 4.633581, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 91.045, "latency": 492.62, "stderr": 1.747, "cost_per_test": 1.070964, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "tencent/hy4-preview": { "accuracy": 90.485, "latency": 3892.496, "stderr": 1.796, "cost_per_test": 0.858064, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-luna": { "accuracy": 90.299, "latency": 1218.504, "stderr": 1.811, "cost_per_test": 0.239206, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-3": { "accuracy": 90.112, "latency": 548.724, "stderr": 1.827, "cost_per_test": 0.087277, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "reflection/Beam-1.0": { "accuracy": 89.179, "latency": 116.862, "stderr": 1.901, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 88.433, "latency": 445.262, "stderr": 1.957, "cost_per_test": 0.169997, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse-spark-open": { "accuracy": 78.545, "latency": 17997.52, "stderr": 2.512, "cost_per_test": 18.435543, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 74.813, "latency": 1273.962, "stderr": 2.657, "cost_per_test": 0.604563, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 74.813, "latency": 3837.702, "stderr": 2.657, "cost_per_test": 3.38767, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 74.254, "latency": 3077.896, "stderr": 2.676, "cost_per_test": 23.532694, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "criteria_pass_rate__bankruptcy_restructuring": { "google/gemini-4-argon": { "accuracy": 95.156, "latency": 1549.322, "stderr": 1.265, "cost_per_test": 10.336011, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 94.983, "latency": 2901.7, "stderr": 1.286, "cost_per_test": 6.272163, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-spark-open": { "accuracy": 94.637, "latency": 19897.52, "stderr": 1.328, "cost_per_test": 10.090618, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse-wonder": { "accuracy": 94.464, "latency": 945.124, "stderr": 1.348, "cost_per_test": 7.317233, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 93.426, "latency": 1285.442, "stderr": 1.46, "cost_per_test": 0.640206, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 93.426, "latency": 6164.394, "stderr": 1.46, "cost_per_test": 5.241005, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 92.734, "latency": 3234.16, "stderr": 1.53, "cost_per_test": 3.947315, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 92.561, "latency": 2160.01, "stderr": 1.546, "cost_per_test": 0.309228, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "kimi/kimi-k3": { "accuracy": 91.869, "latency": 3395.826, "stderr": 1.611, "cost_per_test": 5.810019, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3": { "accuracy": 91.522, "latency": 2053.988, "stderr": 1.641, "cost_per_test": 4.33963, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 91.176, "latency": 1267.312, "stderr": 1.671, "cost_per_test": 4.332858, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 90.138, "latency": 2192.152, "stderr": 1.757, "cost_per_test": 12.833742, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 89.619, "latency": 2674.52, "stderr": 1.797, "cost_per_test": 0.116291, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 89.446, "latency": 1198.624, "stderr": 1.81, "cost_per_test": 1.003903, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 88.408, "latency": 779.568, "stderr": 1.886, "cost_per_test": 0.396682, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 87.716, "latency": 1586.546, "stderr": 1.934, "cost_per_test": 0.501587, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 87.543, "latency": 3463.16, "stderr": 1.946, "cost_per_test": 22.051917, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 87.37, "latency": 2414.9, "stderr": 1.957, "cost_per_test": 3.872355, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 87.37, "latency": 2985.2, "stderr": 1.957, "cost_per_test": 5.081934, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "inception/mercury-3": { "accuracy": 86.678, "latency": 969.208, "stderr": 2.002, "cost_per_test": 0.211254, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-6-astra": { "accuracy": 86.678, "latency": 1968.368, "stderr": 2.002, "cost_per_test": 26.126342, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 85.813, "latency": 96.752, "stderr": 2.056, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "tencent/hy4-preview": { "accuracy": 85.64, "latency": 2261.908, "stderr": 2.066, "cost_per_test": 1.145459, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-luna": { "accuracy": 85.294, "latency": 1306.948, "stderr": 2.087, "cost_per_test": 0.315042, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 75.779, "latency": 491.618, "stderr": 2.525, "cost_per_test": 0.153217, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 75.606, "latency": 3911.966, "stderr": 2.531, "cost_per_test": 15.129321, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 70.588, "latency": 773.418, "stderr": 2.685, "cost_per_test": 1.383205, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__capital_markets": { "meta/muse-spark-open": { "accuracy": 97.843, "latency": 7272.288, "stderr": 0.912, "cost_per_test": 11.942399, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 96.667, "latency": 1403.692, "stderr": 1.126, "cost_per_test": 9.03284, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 96.471, "latency": 1186.85, "stderr": 1.158, "cost_per_test": 2.819389, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 96.471, "latency": 2173.218, "stderr": 1.158, "cost_per_test": 9.948149, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-wonder": { "accuracy": 96.275, "latency": 749.014, "stderr": 1.188, "cost_per_test": 7.059417, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "openai/gpt-6-astra": { "accuracy": 96.078, "latency": 1505.512, "stderr": 1.218, "cost_per_test": 20.140295, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 96.078, "latency": 2353.468, "stderr": 1.218, "cost_per_test": 4.235198, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 95.882, "latency": 2223.106, "stderr": 1.247, "cost_per_test": 3.753987, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 95.49, "latency": 895.892, "stderr": 1.302, "cost_per_test": 0.073201, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 95.098, "latency": 1449.372, "stderr": 1.355, "cost_per_test": 2.151201, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 94.902, "latency": 2232.706, "stderr": 1.38, "cost_per_test": 2.776428, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "kimi/kimi-k3": { "accuracy": 94.51, "latency": 1822.032, "stderr": 1.429, "cost_per_test": 2.677304, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "zai/glm-5.3": { "accuracy": 94.314, "latency": 1684.458, "stderr": 1.453, "cost_per_test": 3.571137, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 94.314, "latency": 1917.712, "stderr": 1.453, "cost_per_test": 2.741122, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 94.118, "latency": 1270.292, "stderr": 1.476, "cost_per_test": 0.604559, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 93.725, "latency": 2995.246, "stderr": 1.522, "cost_per_test": 20.393428, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 93.333, "latency": 1616.54, "stderr": 1.565, "cost_per_test": 0.197788, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "inception/mercury-3": { "accuracy": 93.137, "latency": 396.136, "stderr": 1.586, "cost_per_test": 0.094975, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "tencent/hy4-preview": { "accuracy": 92.941, "latency": 1235.536, "stderr": 1.607, "cost_per_test": 0.791921, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 92.549, "latency": 442.964, "stderr": 1.648, "cost_per_test": 0.628074, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 92.353, "latency": 1222.418, "stderr": 1.667, "cost_per_test": 0.336279, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-luna": { "accuracy": 91.569, "latency": 1059.256, "stderr": 1.743, "cost_per_test": 0.220062, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 91.373, "latency": 538.678, "stderr": 1.762, "cost_per_test": 0.198307, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 89.608, "latency": 97.144, "stderr": 1.915, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 86.471, "latency": 415.398, "stderr": 2.146, "cost_per_test": 0.158893, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 83.529, "latency": 3458.518, "stderr": 2.327, "cost_per_test": 14.684627, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 73.922, "latency": 337.696, "stderr": 2.755, "cost_per_test": 1.181787, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__corporate_governance": { "google/gemini-4-argon": { "accuracy": 97.945, "latency": 1910.844, "stderr": 0.832, "cost_per_test": 7.288851, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 97.603, "latency": 869.038, "stderr": 0.897, "cost_per_test": 5.385755, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 96.747, "latency": 2713.342, "stderr": 1.04, "cost_per_test": 5.247151, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 96.062, "latency": 17034.724, "stderr": 1.14, "cost_per_test": 26.592215, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 95.205, "latency": 2396.124, "stderr": 1.252, "cost_per_test": 3.607788, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 94.692, "latency": 2735.374, "stderr": 1.314, "cost_per_test": 5.218937, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 94.349, "latency": 1900.05, "stderr": 1.354, "cost_per_test": 4.113932, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 93.836, "latency": 1100.274, "stderr": 1.41, "cost_per_test": 4.21486, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 93.493, "latency": 2584.122, "stderr": 1.446, "cost_per_test": 3.108266, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 92.979, "latency": 1757.566, "stderr": 1.498, "cost_per_test": 0.202579, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "reflection/Beam-1.0": { "accuracy": 92.808, "latency": 140.722, "stderr": 1.514, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 92.637, "latency": 566.566, "stderr": 1.531, "cost_per_test": 0.759983, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 91.952, "latency": 2408.28, "stderr": 1.595, "cost_per_test": 4.104081, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 91.438, "latency": 2097.538, "stderr": 1.64, "cost_per_test": 0.628725, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "grok/grok-4.7": { "accuracy": 91.438, "latency": 2099.78, "stderr": 1.64, "cost_per_test": 11.205219, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 91.438, "latency": 5172.942, "stderr": 1.64, "cost_per_test": 16.225332, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 90.925, "latency": 1100.848, "stderr": 1.684, "cost_per_test": 0.07785, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "inception/mercury-3": { "accuracy": 90.582, "latency": 1257.38, "stderr": 1.712, "cost_per_test": 0.319761, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 90.582, "latency": 1497.44, "stderr": 1.712, "cost_per_test": 0.70205, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 89.384, "latency": 1893.436, "stderr": 1.806, "cost_per_test": 28.949058, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 89.041, "latency": 1245.05, "stderr": 1.831, "cost_per_test": 0.225592, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 88.527, "latency": 622.074, "stderr": 1.868, "cost_per_test": 0.264988, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 87.5, "latency": 534.078, "stderr": 1.939, "cost_per_test": 0.200642, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "tencent/hy4-preview": { "accuracy": 87.329, "latency": 1770.524, "stderr": 1.95, "cost_per_test": 1.030237, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 87.158, "latency": 2758.188, "stderr": 1.961, "cost_per_test": 4.581533, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 85.616, "latency": 3589.944, "stderr": 2.057, "cost_per_test": 24.770063, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 83.904, "latency": 507.184, "stderr": 2.154, "cost_per_test": 1.493244, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__corporate_ma": { "meta/muse-wonder": { "accuracy": 97.91, "latency": 894.054, "stderr": 0.812, "cost_per_test": 7.82908, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 97.91, "latency": 2372.524, "stderr": 0.812, "cost_per_test": 3.938856, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 97.588, "latency": 2894.524, "stderr": 0.871, "cost_per_test": 5.165325, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 97.267, "latency": 1717.438, "stderr": 0.926, "cost_per_test": 8.132612, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 96.945, "latency": 1001.998, "stderr": 0.977, "cost_per_test": 3.550379, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 96.945, "latency": 2751.628, "stderr": 0.977, "cost_per_test": 3.053264, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 96.463, "latency": 1133.996, "stderr": 1.049, "cost_per_test": 0.07413, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "tencent/hy4-preview": { "accuracy": 96.463, "latency": 1940.384, "stderr": 1.049, "cost_per_test": 0.94797, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 96.463, "latency": 3267.648, "stderr": 1.049, "cost_per_test": 24.674742, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 96.302, "latency": 1048.75, "stderr": 1.072, "cost_per_test": 0.385034, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "zai/glm-5.3": { "accuracy": 96.302, "latency": 1892.662, "stderr": 1.072, "cost_per_test": 4.298734, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 95.82, "latency": 1697.334, "stderr": 1.137, "cost_per_test": 0.176755, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 95.659, "latency": 4191.198, "stderr": 1.157, "cost_per_test": 15.021633, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 95.659, "latency": 13513.444, "stderr": 1.157, "cost_per_test": 11.299235, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "grok/grok-4.7": { "accuracy": 95.498, "latency": 2005.262, "stderr": 1.178, "cost_per_test": 8.576322, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 95.338, "latency": 1348.014, "stderr": 1.197, "cost_per_test": 0.64235, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 95.177, "latency": 587.404, "stderr": 1.217, "cost_per_test": 0.785052, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 94.855, "latency": 662.998, "stderr": 1.255, "cost_per_test": 0.233836, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 94.855, "latency": 2093.188, "stderr": 1.255, "cost_per_test": 22.768442, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 94.695, "latency": 2164.786, "stderr": 1.273, "cost_per_test": 3.081992, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 94.212, "latency": 2442.158, "stderr": 1.326, "cost_per_test": 3.48516, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-3": { "accuracy": 93.248, "latency": 785.954, "stderr": 1.425, "cost_per_test": 0.208695, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 93.248, "latency": 2321.544, "stderr": 1.425, "cost_per_test": 4.04342, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 92.926, "latency": 113.714, "stderr": 1.456, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 92.444, "latency": 369.67, "stderr": 1.501, "cost_per_test": 0.181179, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-luna": { "accuracy": 92.122, "latency": 1532.814, "stderr": 1.53, "cost_per_test": 0.274244, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 79.904, "latency": 456.474, "stderr": 2.276, "cost_per_test": 1.157097, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__data_privacy_cybersecurity": { "google/gemini-4-argon": { "accuracy": 98.41, "latency": 1849.552, "stderr": 0.745, "cost_per_test": 8.325019, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 98.233, "latency": 2038.44, "stderr": 0.785, "cost_per_test": 13.898195, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "meta/muse-spark-open": { "accuracy": 98.233, "latency": 8016.45, "stderr": 0.785, "cost_per_test": 6.234146, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 98.057, "latency": 2553.994, "stderr": 0.822, "cost_per_test": 5.724329, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 97.88, "latency": 2027.386, "stderr": 0.858, "cost_per_test": 6.026136, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "meta/muse-wonder": { "accuracy": 97.703, "latency": 830.088, "stderr": 0.892, "cost_per_test": 6.429998, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 97.703, "latency": 1600.072, "stderr": 0.892, "cost_per_test": 0.204203, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 97.703, "latency": 2644.256, "stderr": 0.892, "cost_per_test": 3.560927, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 97.527, "latency": 1228.252, "stderr": 0.925, "cost_per_test": 0.52666, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 97.527, "latency": 1295.87, "stderr": 0.925, "cost_per_test": 2.439638, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 97.173, "latency": 2086.04, "stderr": 0.987, "cost_per_test": 4.139175, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 97.173, "latency": 2631.3, "stderr": 0.987, "cost_per_test": 5.085316, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "inception/mercury-3": { "accuracy": 96.82, "latency": 803.208, "stderr": 1.045, "cost_per_test": 0.159261, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 96.82, "latency": 1603.698, "stderr": 1.045, "cost_per_test": 0.652971, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 96.643, "latency": 3116.616, "stderr": 1.073, "cost_per_test": 21.050628, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 96.466, "latency": 532.25, "stderr": 1.099, "cost_per_test": 0.795228, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 96.466, "latency": 1114.628, "stderr": 1.099, "cost_per_test": 0.083103, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 96.29, "latency": 2375.93, "stderr": 1.126, "cost_per_test": 3.908605, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 95.76, "latency": 2134.04, "stderr": 1.2, "cost_per_test": 32.097478, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 95.406, "latency": 2352.11, "stderr": 1.247, "cost_per_test": 3.758665, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 95.23, "latency": 702.086, "stderr": 1.269, "cost_per_test": 0.331272, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 95.23, "latency": 1542.326, "stderr": 1.269, "cost_per_test": 1.156317, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "reflection/Beam-1.0": { "accuracy": 93.11, "latency": 120.522, "stderr": 1.508, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "openai/gpt-6-luna": { "accuracy": 91.519, "latency": 1319.848, "stderr": 1.659, "cost_per_test": 0.292544, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 84.806, "latency": 401.836, "stderr": 2.138, "cost_per_test": 0.203808, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 59.364, "latency": 568.356, "stderr": 2.925, "cost_per_test": 1.636783, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 59.364, "latency": 4080.704, "stderr": 2.925, "cost_per_test": 16.205227, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "criteria_pass_rate__emerging_companies_venture_capital": { "meta/muse-wonder": { "accuracy": 97.44, "latency": 972.262, "stderr": 0.924, "cost_per_test": 7.098804, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 97.099, "latency": 2502.37, "stderr": 0.982, "cost_per_test": 6.394651, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 96.416, "latency": 1468.188, "stderr": 1.088, "cost_per_test": 13.959833, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 95.563, "latency": 4124.448, "stderr": 1.205, "cost_per_test": 16.606099, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "grok/grok-4.7": { "accuracy": 95.222, "latency": 2252.824, "stderr": 1.248, "cost_per_test": 9.503463, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 94.881, "latency": 1539.008, "stderr": 1.29, "cost_per_test": 0.633521, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 94.539, "latency": 1523.606, "stderr": 1.33, "cost_per_test": 0.437811, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "meta/muse-spark-open": { "accuracy": 94.198, "latency": 20320.366, "stderr": 1.368, "cost_per_test": 18.393531, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-5.3": { "accuracy": 93.686, "latency": 2229.252, "stderr": 1.423, "cost_per_test": 4.599163, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 93.686, "latency": 3632.004, "stderr": 1.423, "cost_per_test": 4.022647, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 93.515, "latency": 2035.666, "stderr": 1.441, "cost_per_test": 0.222921, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 93.345, "latency": 618.824, "stderr": 1.459, "cost_per_test": 0.819599, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 93.003, "latency": 2231.368, "stderr": 1.493, "cost_per_test": 4.18071, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 92.833, "latency": 3178.396, "stderr": 1.51, "cost_per_test": 19.621784, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 92.491, "latency": 2505.308, "stderr": 1.542, "cost_per_test": 4.158411, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 92.15, "latency": 769.368, "stderr": 1.574, "cost_per_test": 0.28403, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 92.15, "latency": 1360.808, "stderr": 1.574, "cost_per_test": 0.274165, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 92.15, "latency": 1362.822, "stderr": 1.574, "cost_per_test": 0.990763, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "kimi/kimi-k3": { "accuracy": 92.15, "latency": 2548.358, "stderr": 1.574, "cost_per_test": 3.154588, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 91.98, "latency": 2707.794, "stderr": 1.589, "cost_per_test": 3.778376, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 91.809, "latency": 2030.458, "stderr": 1.605, "cost_per_test": 20.81811, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 91.468, "latency": 1103.358, "stderr": 1.635, "cost_per_test": 0.078418, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "reflection/Beam-1.0": { "accuracy": 88.737, "latency": 143.092, "stderr": 1.85, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "inception/mercury-3": { "accuracy": 87.884, "latency": 955.938, "stderr": 1.91, "cost_per_test": 0.148034, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 74.915, "latency": 946.74, "stderr": 2.537, "cost_per_test": 3.954288, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 74.744, "latency": 449.274, "stderr": 2.543, "cost_per_test": 0.200938, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 67.577, "latency": 744.508, "stderr": 2.739, "cost_per_test": 1.206641, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__employment_labor": { "meta/muse-wonder-full": { "accuracy": 93.446, "latency": 2510.782, "stderr": 1.517, "cost_per_test": 4.65707, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 93.071, "latency": 862.152, "stderr": 1.557, "cost_per_test": 7.283077, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 92.884, "latency": 1416.794, "stderr": 1.576, "cost_per_test": 6.38748, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 92.509, "latency": 16944.098, "stderr": 1.614, "cost_per_test": 12.206492, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 90.637, "latency": 1291.326, "stderr": 1.786, "cost_per_test": 0.520216, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "grok/grok-4.7": { "accuracy": 90.075, "latency": 2245.098, "stderr": 1.833, "cost_per_test": 9.137915, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 89.326, "latency": 1128.258, "stderr": 1.893, "cost_per_test": 0.5861, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 89.326, "latency": 2745.574, "stderr": 1.893, "cost_per_test": 2.560531, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "openai/gpt-6-astra": { "accuracy": 89.139, "latency": 1778.602, "stderr": 1.908, "cost_per_test": 14.13358, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 89.139, "latency": 4514.362, "stderr": 1.908, "cost_per_test": 2.622256, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-5.3": { "accuracy": 88.764, "latency": 1861.322, "stderr": 1.936, "cost_per_test": 2.739975, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 88.764, "latency": 2750.208, "stderr": 1.936, "cost_per_test": 3.430645, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 88.577, "latency": 1108.248, "stderr": 1.95, "cost_per_test": 3.168674, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 88.577, "latency": 2499.984, "stderr": 1.95, "cost_per_test": 0.067578, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 88.202, "latency": 2219.336, "stderr": 1.978, "cost_per_test": 0.176119, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 88.015, "latency": 2415.098, "stderr": 1.991, "cost_per_test": 2.463712, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 87.266, "latency": 2478.518, "stderr": 2.044, "cost_per_test": 3.640709, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 86.33, "latency": 1990.538, "stderr": 2.106, "cost_per_test": 0.881184, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 85.206, "latency": 3164.882, "stderr": 2.177, "cost_per_test": 14.514568, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 84.644, "latency": 631.722, "stderr": 2.211, "cost_per_test": 0.188769, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-3": { "accuracy": 84.27, "latency": 400.446, "stderr": 2.232, "cost_per_test": 0.143202, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-6-luna": { "accuracy": 83.708, "latency": 1239.982, "stderr": 2.264, "cost_per_test": 0.165124, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 83.521, "latency": 123.484, "stderr": 2.275, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 82.21, "latency": 1855.334, "stderr": 2.345, "cost_per_test": 0.426654, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 80.15, "latency": 4200.754, "stderr": 2.446, "cost_per_test": 17.260921, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 79.401, "latency": 474.796, "stderr": 2.48, "cost_per_test": 1.781902, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 77.903, "latency": 495.416, "stderr": 2.544, "cost_per_test": 0.197427, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__energy_natural_resources": { "meta/muse-wonder": { "accuracy": 98.515, "latency": 1067.054, "stderr": 0.696, "cost_per_test": 11.537685, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 98.515, "latency": 2350.706, "stderr": 0.696, "cost_per_test": 14.055481, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 98.185, "latency": 3120.71, "stderr": 0.768, "cost_per_test": 6.264605, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-spark-open": { "accuracy": 98.185, "latency": 17549.494, "stderr": 0.768, "cost_per_test": 22.510187, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 98.02, "latency": 1272.778, "stderr": 0.802, "cost_per_test": 0.69079, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 97.855, "latency": 548.636, "stderr": 0.834, "cost_per_test": 0.979999, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3": { "accuracy": 97.855, "latency": 1863.802, "stderr": 0.834, "cost_per_test": 5.176944, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 97.525, "latency": 621.178, "stderr": 0.894, "cost_per_test": 0.33458, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 97.525, "latency": 1422.04, "stderr": 0.894, "cost_per_test": 4.405884, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 97.525, "latency": 3576.094, "stderr": 0.894, "cost_per_test": 7.569589, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 97.36, "latency": 3212.248, "stderr": 0.923, "cost_per_test": 4.853818, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "grok/grok-4.7": { "accuracy": 96.865, "latency": 1822.396, "stderr": 1.003, "cost_per_test": 11.371091, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 96.535, "latency": 2661.574, "stderr": 1.052, "cost_per_test": 3.699196, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 96.535, "latency": 2751.118, "stderr": 1.052, "cost_per_test": 5.971747, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 96.37, "latency": 2434.558, "stderr": 1.076, "cost_per_test": 52.676518, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 96.205, "latency": 1592.142, "stderr": 1.1, "cost_per_test": 0.125628, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 95.875, "latency": 1434.758, "stderr": 1.144, "cost_per_test": 0.616064, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 95.71, "latency": 1924.146, "stderr": 1.166, "cost_per_test": 0.288516, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-luna": { "accuracy": 95.38, "latency": 1398.494, "stderr": 1.208, "cost_per_test": 0.576369, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 94.389, "latency": 101.376, "stderr": 1.324, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "inception/mercury-3": { "accuracy": 94.224, "latency": 1050.646, "stderr": 1.342, "cost_per_test": 0.277782, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 93.564, "latency": 2951.16, "stderr": 1.412, "cost_per_test": 5.570413, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 92.739, "latency": 2322.328, "stderr": 1.493, "cost_per_test": 1.207559, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 91.749, "latency": 3915.456, "stderr": 1.583, "cost_per_test": 26.383042, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 79.868, "latency": 318.21, "stderr": 2.307, "cost_per_test": 0.186124, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 59.571, "latency": 5117.248, "stderr": 2.824, "cost_per_test": 17.609237, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 32.673, "latency": 231.65, "stderr": 2.699, "cost_per_test": 0.939334, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__environmental_esg": { "google/gemini-4-argon": { "accuracy": 97.312, "latency": 1868.158, "stderr": 0.97, "cost_per_test": 8.278315, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 94.803, "latency": 820.33, "stderr": 1.331, "cost_per_test": 5.178019, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-spark-open": { "accuracy": 93.728, "latency": 9213.876, "stderr": 1.454, "cost_per_test": 12.590654, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 93.19, "latency": 2908.808, "stderr": 1.511, "cost_per_test": 4.77579, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 93.011, "latency": 1433.79, "stderr": 1.529, "cost_per_test": 0.635298, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 92.473, "latency": 1210.54, "stderr": 1.582, "cost_per_test": 3.219091, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 91.935, "latency": 861.556, "stderr": 1.633, "cost_per_test": 0.300222, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3": { "accuracy": 91.935, "latency": 2625.67, "stderr": 1.633, "cost_per_test": 3.885641, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "grok/grok-4.7": { "accuracy": 91.039, "latency": 2350.754, "stderr": 1.713, "cost_per_test": 10.857757, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 91.039, "latency": 3014.468, "stderr": 1.713, "cost_per_test": 3.563589, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 90.681, "latency": 3321.142, "stderr": 1.743, "cost_per_test": 2.910771, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 90.502, "latency": 1119.692, "stderr": 1.758, "cost_per_test": 0.086836, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 89.785, "latency": 678.776, "stderr": 1.816, "cost_per_test": 0.727139, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "kimi/kimi-k3": { "accuracy": 89.785, "latency": 2824.202, "stderr": 1.816, "cost_per_test": 3.651165, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 89.606, "latency": 1976.582, "stderr": 1.83, "cost_per_test": 17.922383, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 89.427, "latency": 1706.268, "stderr": 1.844, "cost_per_test": 0.440838, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 89.427, "latency": 2278.048, "stderr": 1.844, "cost_per_test": 0.232534, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 88.889, "latency": 4846.184, "stderr": 1.885, "cost_per_test": 18.079139, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 88.351, "latency": 2880.8, "stderr": 1.924, "cost_per_test": 4.552595, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 88.172, "latency": 2681.314, "stderr": 1.937, "cost_per_test": 3.292441, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 87.993, "latency": 145.024, "stderr": 1.949, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "openai/gpt-6-luna": { "accuracy": 87.634, "latency": 1205.648, "stderr": 1.974, "cost_per_test": 0.156612, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 86.559, "latency": 1835.898, "stderr": 2.046, "cost_per_test": 0.897665, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 86.201, "latency": 3643.502, "stderr": 2.069, "cost_per_test": 24.257857, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "inception/mercury-3": { "accuracy": 85.305, "latency": 1093.524, "stderr": 2.124, "cost_per_test": 0.24568, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "thinkingmachines/inkling": { "accuracy": 84.767, "latency": 535.646, "stderr": 2.155, "cost_per_test": 1.189505, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 76.165, "latency": 506.562, "stderr": 2.555, "cost_per_test": 0.17309, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__funds_asset_management": { "meta/muse-wonder": { "accuracy": 95.044, "latency": 1090.864, "stderr": 1.174, "cost_per_test": 12.604977, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 93.878, "latency": 3311.582, "stderr": 1.296, "cost_per_test": 6.091472, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 93.003, "latency": 2070.686, "stderr": 1.379, "cost_per_test": 10.601161, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 91.254, "latency": 4340.334, "stderr": 1.528, "cost_per_test": 5.381658, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 91.108, "latency": 3291.178, "stderr": 1.539, "cost_per_test": 5.61229, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.7": { "accuracy": 90.816, "latency": 2190.716, "stderr": 1.562, "cost_per_test": 12.657496, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 90.525, "latency": 1365.906, "stderr": 1.584, "cost_per_test": 3.568526, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 89.942, "latency": 2848.396, "stderr": 1.626, "cost_per_test": 3.952966, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 89.796, "latency": 1356.48, "stderr": 1.637, "cost_per_test": 0.654524, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 89.796, "latency": 3871.644, "stderr": 1.637, "cost_per_test": 5.123026, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 89.504, "latency": 1849.72, "stderr": 1.657, "cost_per_test": 4.143899, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 88.63, "latency": 2500.89, "stderr": 1.717, "cost_per_test": 0.275778, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6-astra": { "accuracy": 88.484, "latency": 2380.55, "stderr": 1.726, "cost_per_test": 36.827844, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 88.047, "latency": 727.476, "stderr": 1.754, "cost_per_test": 1.084471, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 87.901, "latency": 1378.846, "stderr": 1.763, "cost_per_test": 0.106311, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 87.901, "latency": 1901.926, "stderr": 1.763, "cost_per_test": 0.565023, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 87.318, "latency": 785.382, "stderr": 1.799, "cost_per_test": 0.344122, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 87.318, "latency": 4100.024, "stderr": 1.799, "cost_per_test": 21.45996, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 87.318, "latency": 5031.938, "stderr": 1.799, "cost_per_test": 19.376575, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 85.714, "latency": 2055.006, "stderr": 1.892, "cost_per_test": 1.20772, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 85.569, "latency": 3055.95, "stderr": 1.9, "cost_per_test": 4.810012, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "inception/mercury-3": { "accuracy": 84.985, "latency": 1470.158, "stderr": 1.932, "cost_per_test": 0.360777, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-6-luna": { "accuracy": 83.236, "latency": 1405.04, "stderr": 2.02, "cost_per_test": 0.293666, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 80.612, "latency": 125.446, "stderr": 2.138, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 80.466, "latency": 468.998, "stderr": 2.144, "cost_per_test": 0.160247, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse-spark-open": { "accuracy": 74.927, "latency": 18279.368, "stderr": 2.344, "cost_per_test": 14.222782, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 66.327, "latency": 720.322, "stderr": 2.555, "cost_per_test": 1.427877, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__healthcare_life_sciences": { "google/gemini-4-argon": { "accuracy": 97.819, "latency": 2163.428, "stderr": 0.848, "cost_per_test": 8.843854, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 97.315, "latency": 2360.19, "stderr": 0.938, "cost_per_test": 3.296002, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 96.812, "latency": 2431.536, "stderr": 1.019, "cost_per_test": 4.871642, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 96.812, "latency": 3257.984, "stderr": 1.019, "cost_per_test": 3.63425, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "meta/muse-wonder-full": { "accuracy": 96.644, "latency": 2539.434, "stderr": 1.045, "cost_per_test": 4.848495, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 96.141, "latency": 1854.082, "stderr": 1.118, "cost_per_test": 3.150292, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 95.973, "latency": 802.956, "stderr": 1.141, "cost_per_test": 0.469585, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 95.973, "latency": 1457.208, "stderr": 1.141, "cost_per_test": 0.656838, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 95.973, "latency": 1642.936, "stderr": 1.141, "cost_per_test": 0.21495, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 95.302, "latency": 922.306, "stderr": 1.228, "cost_per_test": 0.09248, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "meta/muse-wonder": { "accuracy": 95.134, "latency": 975.282, "stderr": 1.248, "cost_per_test": 5.625299, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 95.134, "latency": 1851.156, "stderr": 1.248, "cost_per_test": 0.526994, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "grok/grok-4.7": { "accuracy": 95.134, "latency": 2275.232, "stderr": 1.248, "cost_per_test": 11.67739, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 94.631, "latency": 1125.866, "stderr": 1.308, "cost_per_test": 3.474289, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 94.128, "latency": 5352.558, "stderr": 1.364, "cost_per_test": 19.894952, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 93.624, "latency": 115.612, "stderr": 1.418, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "inception/mercury-3": { "accuracy": 93.289, "latency": 769.502, "stderr": 1.452, "cost_per_test": 0.17794, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-6-astra": { "accuracy": 93.289, "latency": 2205.706, "stderr": 1.452, "cost_per_test": 22.288291, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 92.785, "latency": 4038.166, "stderr": 1.501, "cost_per_test": 28.864668, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 92.114, "latency": 581.14, "stderr": 1.564, "cost_per_test": 0.741568, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 91.946, "latency": 2608.918, "stderr": 1.579, "cost_per_test": 3.982873, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 91.611, "latency": 2990.288, "stderr": 1.609, "cost_per_test": 3.628876, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 91.443, "latency": 1760.07, "stderr": 1.623, "cost_per_test": 1.267174, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 90.772, "latency": 528.81, "stderr": 1.679, "cost_per_test": 0.199914, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-luna": { "accuracy": 87.081, "latency": 1362.432, "stderr": 1.946, "cost_per_test": 0.267572, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "meta/muse-spark-open": { "accuracy": 73.993, "latency": 9853.696, "stderr": 2.545, "cost_per_test": 16.770892, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "thinkingmachines/inkling": { "accuracy": 40.268, "latency": 830.858, "stderr": 2.846, "cost_per_test": 1.476459, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__immigration": { "meta/muse-wonder": { "accuracy": 96.182, "latency": 622.17, "stderr": 1.158, "cost_per_test": 3.50291, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 95.455, "latency": 2604.304, "stderr": 1.258, "cost_per_test": 4.272067, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 94.545, "latency": 1343.676, "stderr": 1.372, "cost_per_test": 9.272635, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 94.364, "latency": 2388.162, "stderr": 1.393, "cost_per_test": 9.158315, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 94.0, "latency": 2807.95, "stderr": 1.435, "cost_per_test": 2.289267, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 94.0, "latency": 3787.048, "stderr": 1.435, "cost_per_test": 13.84088, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "zai/glm-5.3": { "accuracy": 93.091, "latency": 1749.038, "stderr": 1.532, "cost_per_test": 2.461041, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 92.545, "latency": 1218.928, "stderr": 1.587, "cost_per_test": 0.468313, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 92.0, "latency": 1107.642, "stderr": 1.639, "cost_per_test": 2.608087, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "tencent/hy4-preview": { "accuracy": 92.0, "latency": 1376.798, "stderr": 1.639, "cost_per_test": 0.79121, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "openai/gpt-6-astra": { "accuracy": 92.0, "latency": 1584.174, "stderr": 1.639, "cost_per_test": 11.992363, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 92.0, "latency": 2070.532, "stderr": 1.639, "cost_per_test": 2.772501, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 91.818, "latency": 3151.272, "stderr": 1.656, "cost_per_test": 18.52264, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 91.455, "latency": 1745.378, "stderr": 1.689, "cost_per_test": 1.539912, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 91.273, "latency": 1942.628, "stderr": 1.705, "cost_per_test": 2.10625, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 91.091, "latency": 1779.824, "stderr": 1.721, "cost_per_test": 0.178109, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 90.909, "latency": 899.282, "stderr": 1.737, "cost_per_test": 0.069274, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 90.909, "latency": 2118.592, "stderr": 1.737, "cost_per_test": 2.855998, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 90.727, "latency": 1131.4, "stderr": 1.752, "cost_per_test": 0.37483, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 90.182, "latency": 663.922, "stderr": 1.798, "cost_per_test": 0.2425, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 88.364, "latency": 483.414, "stderr": 1.937, "cost_per_test": 0.526006, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "reflection/Beam-1.0": { "accuracy": 87.273, "latency": 114.494, "stderr": 2.013, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling": { "accuracy": 87.273, "latency": 362.142, "stderr": 2.013, "cost_per_test": 0.851971, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-3": { "accuracy": 87.273, "latency": 659.556, "stderr": 2.013, "cost_per_test": 0.074179, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-6-luna": { "accuracy": 85.636, "latency": 1151.354, "stderr": 2.119, "cost_per_test": 0.130511, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 81.273, "latency": 485.528, "stderr": 2.357, "cost_per_test": 0.15155, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse-spark-open": { "accuracy": 71.273, "latency": 8923.964, "stderr": 2.734, "cost_per_test": 5.294027, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null } }, "criteria_pass_rate__insurance": { "google/gemini-4-argon": { "accuracy": 96.324, "latency": 1446.246, "stderr": 1.143, "cost_per_test": 6.035978, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder": { "accuracy": 96.14, "latency": 921.1, "stderr": 1.17, "cost_per_test": 7.517888, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 95.956, "latency": 2466.874, "stderr": 1.197, "cost_per_test": 4.928353, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 94.853, "latency": 2030.584, "stderr": 1.342, "cost_per_test": 9.139267, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 94.485, "latency": 1109.874, "stderr": 1.387, "cost_per_test": 0.476309, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 94.301, "latency": 1280.274, "stderr": 1.408, "cost_per_test": 3.594349, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 94.301, "latency": 1644.074, "stderr": 1.408, "cost_per_test": 0.168232, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 93.75, "latency": 1284.864, "stderr": 1.47, "cost_per_test": 0.060043, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 93.382, "latency": 1803.898, "stderr": 1.51, "cost_per_test": 2.905753, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 93.199, "latency": 3072.686, "stderr": 1.529, "cost_per_test": 16.375474, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 92.831, "latency": 2472.566, "stderr": 1.567, "cost_per_test": 2.316687, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 92.831, "latency": 3563.796, "stderr": 1.567, "cost_per_test": 15.744184, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "kimi/kimi-k3": { "accuracy": 92.096, "latency": 2051.788, "stderr": 1.639, "cost_per_test": 2.912115, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 91.728, "latency": 533.98, "stderr": 1.673, "cost_per_test": 0.667337, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 91.728, "latency": 1686.474, "stderr": 1.673, "cost_per_test": 14.040593, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 91.544, "latency": 648.634, "stderr": 1.69, "cost_per_test": 0.231492, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "zai/glm-5.3": { "accuracy": 91.544, "latency": 1679.992, "stderr": 1.69, "cost_per_test": 2.315951, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 91.176, "latency": 2009.804, "stderr": 1.723, "cost_per_test": 2.136981, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 90.809, "latency": 2167.052, "stderr": 1.755, "cost_per_test": 3.276103, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 90.441, "latency": 1813.96, "stderr": 1.786, "cost_per_test": 0.879072, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 90.074, "latency": 1028.702, "stderr": 1.816, "cost_per_test": 0.343356, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "reflection/Beam-1.0": { "accuracy": 89.338, "latency": 141.572, "stderr": 1.875, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling": { "accuracy": 88.971, "latency": 351.262, "stderr": 1.903, "cost_per_test": 1.138776, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "inception/mercury-3": { "accuracy": 88.971, "latency": 854.286, "stderr": 1.903, "cost_per_test": 0.088321, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-6-luna": { "accuracy": 85.846, "latency": 980.822, "stderr": 2.117, "cost_per_test": 0.194079, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 82.904, "latency": 220.402, "stderr": 2.287, "cost_per_test": 0.135413, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse-spark-open": { "accuracy": 59.743, "latency": 16077.45, "stderr": 2.979, "cost_per_test": 17.909109, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null } }, "criteria_pass_rate__intellectual_property": { "google/gemini-4-argon": { "accuracy": 98.399, "latency": 1994.312, "stderr": 0.75, "cost_per_test": 10.686247, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 98.043, "latency": 2441.248, "stderr": 0.828, "cost_per_test": 3.435719, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse-wonder-full": { "accuracy": 97.865, "latency": 2474.152, "stderr": 0.864, "cost_per_test": 5.894615, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "zai/glm-5.3": { "accuracy": 97.331, "latency": 1690.696, "stderr": 0.963, "cost_per_test": 3.514176, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 97.331, "latency": 2626.582, "stderr": 0.963, "cost_per_test": 3.255216, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 97.331, "latency": 3302.178, "stderr": 0.963, "cost_per_test": 5.422039, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "meta/muse-wonder": { "accuracy": 97.153, "latency": 818.0, "stderr": 0.994, "cost_per_test": 7.383931, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 96.619, "latency": 4362.09, "stderr": 1.08, "cost_per_test": 17.055475, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 96.263, "latency": 16950.634, "stderr": 1.133, "cost_per_test": 14.304537, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 96.085, "latency": 815.826, "stderr": 1.159, "cost_per_test": 0.359106, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 95.907, "latency": 1122.132, "stderr": 1.184, "cost_per_test": 0.088739, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 95.73, "latency": 978.73, "stderr": 1.208, "cost_per_test": 3.241614, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "inception/mercury-3": { "accuracy": 95.374, "latency": 546.538, "stderr": 1.255, "cost_per_test": 0.135627, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 95.196, "latency": 655.82, "stderr": 1.278, "cost_per_test": 0.858728, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "tencent/hy4-preview": { "accuracy": 95.196, "latency": 1923.054, "stderr": 1.278, "cost_per_test": 1.123387, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "reflection/Beam-1.0": { "accuracy": 95.018, "latency": 109.824, "stderr": 1.3, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 95.018, "latency": 1761.748, "stderr": 1.3, "cost_per_test": 0.205832, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 94.84, "latency": 1311.01, "stderr": 1.322, "cost_per_test": 0.601227, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6-astra": { "accuracy": 94.484, "latency": 1837.104, "stderr": 1.364, "cost_per_test": 19.556639, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 94.484, "latency": 2049.792, "stderr": 1.364, "cost_per_test": 2.395422, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 94.306, "latency": 2175.528, "stderr": 1.385, "cost_per_test": 10.417022, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 93.416, "latency": 3385.322, "stderr": 1.482, "cost_per_test": 20.701364, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 92.349, "latency": 1452.402, "stderr": 1.589, "cost_per_test": 0.496673, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-luna": { "accuracy": 92.171, "latency": 1253.686, "stderr": 1.605, "cost_per_test": 0.259407, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 91.815, "latency": 2882.972, "stderr": 1.638, "cost_per_test": 4.592565, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 86.655, "latency": 441.472, "stderr": 2.032, "cost_per_test": 0.163547, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 72.242, "latency": 702.04, "stderr": 2.676, "cost_per_test": 1.137063, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__international_trade_sanctions": { "meta/muse-wonder-full": { "accuracy": 97.211, "latency": 2063.232, "stderr": 1.041, "cost_per_test": 4.26355, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 96.414, "latency": 902.572, "stderr": 1.176, "cost_per_test": 3.602648, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 96.215, "latency": 1409.552, "stderr": 1.207, "cost_per_test": 7.007018, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "kimi/kimi-k3": { "accuracy": 95.817, "latency": 2164.224, "stderr": 1.266, "cost_per_test": 2.611892, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "grok/grok-4.7": { "accuracy": 95.219, "latency": 2203.268, "stderr": 1.349, "cost_per_test": 9.257126, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 94.821, "latency": 1463.816, "stderr": 1.402, "cost_per_test": 0.612386, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 94.024, "latency": 2588.454, "stderr": 1.499, "cost_per_test": 2.262196, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 93.625, "latency": 1643.282, "stderr": 1.545, "cost_per_test": 2.37339, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 93.426, "latency": 1736.24, "stderr": 1.567, "cost_per_test": 0.192448, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 92.829, "latency": 6327.422, "stderr": 1.632, "cost_per_test": 3.46041, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 92.231, "latency": 4004.068, "stderr": 1.693, "cost_per_test": 16.269947, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 92.032, "latency": 3447.774, "stderr": 1.713, "cost_per_test": 20.677557, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 91.833, "latency": 1014.756, "stderr": 1.732, "cost_per_test": 0.06912, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 91.833, "latency": 1212.502, "stderr": 1.732, "cost_per_test": 3.921448, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "thinkingmachines/inkling": { "accuracy": 91.434, "latency": 581.252, "stderr": 1.77, "cost_per_test": 1.18382, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 91.434, "latency": 1530.654, "stderr": 1.77, "cost_per_test": 0.355021, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "tencent/hy4-preview": { "accuracy": 91.235, "latency": 2343.334, "stderr": 1.788, "cost_per_test": 0.787466, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 90.637, "latency": 639.65, "stderr": 1.842, "cost_per_test": 0.64046, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 90.438, "latency": 2166.526, "stderr": 1.86, "cost_per_test": 1.991433, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 90.239, "latency": 774.376, "stderr": 1.877, "cost_per_test": 0.257926, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 89.841, "latency": 2319.978, "stderr": 1.911, "cost_per_test": 3.414516, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 89.641, "latency": 1668.724, "stderr": 1.927, "cost_per_test": 12.013329, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "inception/mercury-3": { "accuracy": 89.243, "latency": 583.14, "stderr": 1.96, "cost_per_test": 0.086833, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "reflection/Beam-1.0": { "accuracy": 89.044, "latency": 111.886, "stderr": 1.975, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "openai/gpt-6-luna": { "accuracy": 87.849, "latency": 982.246, "stderr": 2.066, "cost_per_test": 0.115275, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 85.458, "latency": 515.1, "stderr": 2.23, "cost_per_test": 0.174373, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "meta/muse-spark-open": { "accuracy": 79.283, "latency": 10378.424, "stderr": 2.563, "cost_per_test": 4.714199, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null } }, "criteria_pass_rate__litigation_dispute_resolution": { "google/gemini-4-argon": { "accuracy": 96.241, "latency": 1810.548, "stderr": 1.168, "cost_per_test": 10.28651, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 95.865, "latency": 15708.348, "stderr": 1.223, "cost_per_test": 14.973506, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "meta/muse-wonder": { "accuracy": 95.113, "latency": 850.952, "stderr": 1.324, "cost_per_test": 10.904622, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 95.113, "latency": 2486.446, "stderr": 1.324, "cost_per_test": 5.721406, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "grok/grok-4.7": { "accuracy": 94.925, "latency": 2256.59, "stderr": 1.348, "cost_per_test": 11.087814, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 94.173, "latency": 1218.222, "stderr": 1.439, "cost_per_test": 4.594796, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 93.797, "latency": 3012.004, "stderr": 1.482, "cost_per_test": 3.511846, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 93.797, "latency": 5728.15, "stderr": 1.482, "cost_per_test": 4.311466, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "zai/glm-5.3": { "accuracy": 93.421, "latency": 1897.458, "stderr": 1.523, "cost_per_test": 3.967749, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 93.233, "latency": 1809.34, "stderr": 1.543, "cost_per_test": 33.102, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 93.045, "latency": 4130.842, "stderr": 1.563, "cost_per_test": 21.148726, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 92.669, "latency": 718.642, "stderr": 1.601, "cost_per_test": 1.209207, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 92.481, "latency": 624.304, "stderr": 1.62, "cost_per_test": 0.817563, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 92.481, "latency": 1129.724, "stderr": 1.62, "cost_per_test": 0.577367, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 92.293, "latency": 2566.032, "stderr": 1.638, "cost_per_test": 4.235701, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 92.105, "latency": 924.466, "stderr": 1.656, "cost_per_test": 0.089255, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 91.917, "latency": 1470.038, "stderr": 1.674, "cost_per_test": 0.48408, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 91.917, "latency": 1784.046, "stderr": 1.674, "cost_per_test": 0.217599, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 91.729, "latency": 2621.26, "stderr": 1.692, "cost_per_test": 4.966411, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 91.541, "latency": 2669.568, "stderr": 1.709, "cost_per_test": 4.340162, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "tencent/hy4-preview": { "accuracy": 90.789, "latency": 1530.146, "stderr": 1.776, "cost_per_test": 0.914139, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 90.602, "latency": 817.96, "stderr": 1.793, "cost_per_test": 0.371947, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "inception/mercury-3": { "accuracy": 90.038, "latency": 1189.146, "stderr": 1.84, "cost_per_test": 0.276764, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-6-luna": { "accuracy": 88.534, "latency": 2139.15, "stderr": 1.957, "cost_per_test": 0.869002, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 88.346, "latency": 121.658, "stderr": 1.971, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 80.639, "latency": 548.538, "stderr": 2.427, "cost_per_test": 0.200138, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 75.0, "latency": 4470.472, "stderr": 2.66, "cost_per_test": 13.912957, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null } }, "criteria_pass_rate__real_estate": { "meta/muse-wonder-full": { "accuracy": 98.397, "latency": 2871.286, "stderr": 0.679, "cost_per_test": 6.578616, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 97.23, "latency": 790.378, "stderr": 0.887, "cost_per_test": 5.727136, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 97.085, "latency": 2112.972, "stderr": 0.91, "cost_per_test": 16.654201, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-spark-open": { "accuracy": 96.501, "latency": 19874.44, "stderr": 0.994, "cost_per_test": 25.105877, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 96.21, "latency": 2926.518, "stderr": 1.033, "cost_per_test": 5.105426, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 95.918, "latency": 2034.836, "stderr": 1.07, "cost_per_test": 12.048195, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 95.918, "latency": 2897.188, "stderr": 1.07, "cost_per_test": 5.263349, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 95.627, "latency": 1120.304, "stderr": 1.106, "cost_per_test": 4.5574, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 95.19, "latency": 2089.27, "stderr": 1.157, "cost_per_test": 27.678718, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 94.898, "latency": 2466.112, "stderr": 1.19, "cost_per_test": 3.079066, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 94.898, "latency": 2713.792, "stderr": 1.19, "cost_per_test": 4.688049, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 94.752, "latency": 1407.424, "stderr": 1.206, "cost_per_test": 0.669758, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 94.606, "latency": 1191.344, "stderr": 1.221, "cost_per_test": 0.101049, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 94.606, "latency": 1838.822, "stderr": 1.221, "cost_per_test": 4.97377, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 94.169, "latency": 1435.644, "stderr": 1.267, "cost_per_test": 0.466666, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 94.023, "latency": 1943.306, "stderr": 1.282, "cost_per_test": 4.477501, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 93.878, "latency": 1748.814, "stderr": 1.296, "cost_per_test": 0.266735, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 93.44, "latency": 726.292, "stderr": 1.339, "cost_per_test": 1.119961, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 93.003, "latency": 733.85, "stderr": 1.379, "cost_per_test": 0.442482, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 92.42, "latency": 1428.066, "stderr": 1.431, "cost_per_test": 0.471568, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 92.128, "latency": 3861.442, "stderr": 1.456, "cost_per_test": 21.779478, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "tencent/hy4-preview": { "accuracy": 91.837, "latency": 1577.124, "stderr": 1.481, "cost_per_test": 1.123838, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "inception/mercury-3": { "accuracy": 90.233, "latency": 946.294, "stderr": 1.605, "cost_per_test": 0.251521, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 88.047, "latency": 5189.844, "stderr": 1.754, "cost_per_test": 16.877102, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 87.318, "latency": 106.536, "stderr": 1.799, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 77.551, "latency": 452.006, "stderr": 2.256, "cost_per_test": 0.140612, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling": { "accuracy": 66.764, "latency": 437.088, "stderr": 2.547, "cost_per_test": 1.394793, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__structured_finance_securitization": { "meta/muse-wonder-full": { "accuracy": 96.129, "latency": 2426.824, "stderr": 1.097, "cost_per_test": 6.8176, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 94.194, "latency": 969.446, "stderr": 1.33, "cost_per_test": 7.587681, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 94.032, "latency": 1931.426, "stderr": 1.348, "cost_per_test": 13.409945, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 93.548, "latency": 4095.526, "stderr": 1.398, "cost_per_test": 19.757045, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "meta/muse-spark-open": { "accuracy": 93.065, "latency": 15069.208, "stderr": 1.445, "cost_per_test": 6.985702, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 92.903, "latency": 1214.128, "stderr": 1.461, "cost_per_test": 0.5812, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 92.742, "latency": 2433.226, "stderr": 1.476, "cost_per_test": 4.677121, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "grok/grok-4.7": { "accuracy": 92.419, "latency": 2069.346, "stderr": 1.506, "cost_per_test": 10.094563, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "kimi/kimi-k3": { "accuracy": 92.258, "latency": 2283.332, "stderr": 1.52, "cost_per_test": 5.207436, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 92.097, "latency": 3832.726, "stderr": 1.535, "cost_per_test": 13.720065, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 91.935, "latency": 2592.83, "stderr": 1.549, "cost_per_test": 2.813511, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3": { "accuracy": 91.613, "latency": 1620.42, "stderr": 1.577, "cost_per_test": 3.680402, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 91.129, "latency": 1073.882, "stderr": 1.617, "cost_per_test": 0.092304, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 90.806, "latency": 2638.612, "stderr": 1.644, "cost_per_test": 3.667821, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 90.645, "latency": 1084.012, "stderr": 1.657, "cost_per_test": 3.302349, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "openai/gpt-6-astra": { "accuracy": 90.323, "latency": 2092.05, "stderr": 1.682, "cost_per_test": 24.947157, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 90.161, "latency": 1527.02, "stderr": 1.694, "cost_per_test": 0.84184, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 88.871, "latency": 517.66, "stderr": 1.789, "cost_per_test": 0.690061, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 88.387, "latency": 786.548, "stderr": 1.823, "cost_per_test": 0.303097, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 88.226, "latency": 1876.594, "stderr": 1.834, "cost_per_test": 0.212322, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 87.581, "latency": 1092.26, "stderr": 1.876, "cost_per_test": 0.430794, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "openai/gpt-6-luna": { "accuracy": 87.258, "latency": 1362.744, "stderr": 1.897, "cost_per_test": 0.326133, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 86.452, "latency": 3025.816, "stderr": 1.947, "cost_per_test": 5.220471, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "reflection/Beam-1.0": { "accuracy": 86.129, "latency": 106.582, "stderr": 1.966, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "inception/mercury-3": { "accuracy": 83.71, "latency": 1222.948, "stderr": 2.101, "cost_per_test": 0.179295, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "thinkingmachines/inkling": { "accuracy": 82.742, "latency": 273.318, "stderr": 2.15, "cost_per_test": 1.246746, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 77.097, "latency": 499.992, "stderr": 2.39, "cost_per_test": 0.181086, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__tax": { "meta/muse-spark-open": { "accuracy": 96.99, "latency": 11961.888, "stderr": 0.99, "cost_per_test": 10.001683, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 94.816, "latency": 1632.154, "stderr": 1.284, "cost_per_test": 10.875562, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "meta/muse-wonder-full": { "accuracy": 92.642, "latency": 2475.476, "stderr": 1.512, "cost_per_test": 5.723511, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 92.475, "latency": 873.104, "stderr": 1.528, "cost_per_test": 5.425917, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 91.472, "latency": 2086.784, "stderr": 1.618, "cost_per_test": 0.247354, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 91.137, "latency": 2328.808, "stderr": 1.646, "cost_per_test": 5.63308, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 90.301, "latency": 2859.728, "stderr": 1.714, "cost_per_test": 5.054813, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 89.632, "latency": 1330.648, "stderr": 1.766, "cost_per_test": 0.096872, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 89.465, "latency": 1262.568, "stderr": 1.778, "cost_per_test": 4.824852, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 89.465, "latency": 4033.49, "stderr": 1.778, "cost_per_test": 6.118001, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 89.298, "latency": 4537.142, "stderr": 1.791, "cost_per_test": 4.442674, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 88.629, "latency": 1144.05, "stderr": 1.839, "cost_per_test": 0.501765, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 87.793, "latency": 110.238, "stderr": 1.896, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 87.793, "latency": 1437.404, "stderr": 1.896, "cost_per_test": 0.703428, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 85.953, "latency": 2757.38, "stderr": 2.013, "cost_per_test": 4.69923, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 85.619, "latency": 774.622, "stderr": 2.033, "cost_per_test": 1.050716, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 85.284, "latency": 2523.416, "stderr": 2.052, "cost_per_test": 3.793071, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 84.783, "latency": 2204.202, "stderr": 2.081, "cost_per_test": 39.15058, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 84.615, "latency": 2535.448, "stderr": 2.09, "cost_per_test": 11.489974, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 84.281, "latency": 1483.114, "stderr": 2.108, "cost_per_test": 0.548613, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "inception/mercury-3": { "accuracy": 82.609, "latency": 911.938, "stderr": 2.196, "cost_per_test": 0.160866, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 81.773, "latency": 3778.528, "stderr": 2.236, "cost_per_test": 17.274333, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 80.435, "latency": 525.194, "stderr": 2.298, "cost_per_test": 0.205472, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "openai/gpt-6-luna": { "accuracy": 79.431, "latency": 1376.274, "stderr": 2.341, "cost_per_test": 0.329521, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 64.047, "latency": 2999.882, "stderr": 2.78, "cost_per_test": 1.092562, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 62.709, "latency": 4877.428, "stderr": 2.801, "cost_per_test": 16.089025, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 51.003, "latency": 430.432, "stderr": 2.896, "cost_per_test": 0.915144, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__trusts_estates_private_client": { "meta/muse-wonder-full": { "accuracy": 96.565, "latency": 2998.392, "stderr": 1.127, "cost_per_test": 4.090553, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder": { "accuracy": 96.183, "latency": 920.036, "stderr": 1.186, "cost_per_test": 7.713584, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "google/gemini-4-argon": { "accuracy": 95.992, "latency": 1572.052, "stderr": 1.214, "cost_per_test": 16.183565, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 95.42, "latency": 1358.138, "stderr": 1.294, "cost_per_test": 0.610376, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 95.42, "latency": 3115.438, "stderr": 1.294, "cost_per_test": 3.818917, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 95.038, "latency": 1226.72, "stderr": 1.344, "cost_per_test": 4.184536, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 95.038, "latency": 1652.888, "stderr": 1.344, "cost_per_test": 0.088474, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 94.466, "latency": 1947.846, "stderr": 1.415, "cost_per_test": 0.23816, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "zai/glm-5.3": { "accuracy": 94.466, "latency": 2222.098, "stderr": 1.415, "cost_per_test": 5.43764, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "kimi/kimi-k3": { "accuracy": 94.275, "latency": 2335.938, "stderr": 1.438, "cost_per_test": 3.927228, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "meta/muse-spark-open": { "accuracy": 94.275, "latency": 13246.212, "stderr": 1.438, "cost_per_test": 15.744338, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "openai/gpt-6-astra": { "accuracy": 93.893, "latency": 1899.114, "stderr": 1.482, "cost_per_test": 23.695343, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-4.7": { "accuracy": 93.321, "latency": 2252.568, "stderr": 1.545, "cost_per_test": 11.951101, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 92.939, "latency": 653.918, "stderr": 1.586, "cost_per_test": 0.89094, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 92.939, "latency": 762.432, "stderr": 1.586, "cost_per_test": 0.369984, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 92.176, "latency": 4217.882, "stderr": 1.662, "cost_per_test": 15.106108, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 91.603, "latency": 2075.326, "stderr": 1.717, "cost_per_test": 5.517077, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 91.603, "latency": 3542.812, "stderr": 1.717, "cost_per_test": 21.781833, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "inception/mercury-3": { "accuracy": 91.221, "latency": 1327.644, "stderr": 1.752, "cost_per_test": 0.274585, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 90.84, "latency": 1956.23, "stderr": 1.786, "cost_per_test": 2.971612, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "tencent/hy4-preview": { "accuracy": 90.458, "latency": 1410.15, "stderr": 1.819, "cost_per_test": 0.915369, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 89.504, "latency": 1486.298, "stderr": 1.897, "cost_per_test": 0.4795, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 89.313, "latency": 2515.052, "stderr": 1.912, "cost_per_test": 3.805345, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-luna": { "accuracy": 88.55, "latency": 1080.394, "stderr": 1.971, "cost_per_test": 0.243579, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 87.595, "latency": 115.176, "stderr": 2.04, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "thinkingmachines/inkling": { "accuracy": 86.45, "latency": 420.692, "stderr": 2.118, "cost_per_test": 1.392685, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 81.489, "latency": 303.938, "stderr": 2.404, "cost_per_test": 0.186361, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } }, "criteria_pass_rate__white_collar_defense_investigations": { "meta/muse-wonder": { "accuracy": 96.97, "latency": 1201.638, "stderr": 0.996, "cost_per_test": 6.121717, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-wonder-full": { "accuracy": 96.633, "latency": 2595.43, "stderr": 1.048, "cost_per_test": 5.453971, "temperature": 1.0, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "meta/muse-spark-open": { "accuracy": 96.128, "latency": 11793.734, "stderr": 1.121, "cost_per_test": 8.882501, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "Together AI", "harness": null }, "google/gemini-4-argon": { "accuracy": 95.455, "latency": 1880.166, "stderr": 1.211, "cost_per_test": 10.940583, "temperature": null, "top_p": null, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "grok/grok-4.7": { "accuracy": 94.781, "latency": 1976.236, "stderr": 1.293, "cost_per_test": 9.182441, "temperature": 0.7, "top_p": 0.95, "max_output_tokens": null, "reasoning": null, "reasoning_effort": "xhigh", "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "zai/glm-5.3": { "accuracy": 94.444, "latency": 2147.146, "stderr": 1.331, "cost_per_test": 4.334048, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "alibaba/qwen3.8-max-internal": { "accuracy": 93.939, "latency": 2749.246, "stderr": 1.387, "cost_per_test": 2.850712, "temperature": 0.7, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Alibaba", "harness": null }, "mistralai/mistral-large-4": { "accuracy": 92.424, "latency": 2455.672, "stderr": 1.538, "cost_per_test": 3.663112, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Mistral AI", "harness": null }, "google/gemini-3.8-flash": { "accuracy": 91.919, "latency": 1076.182, "stderr": 1.584, "cost_per_test": 3.121405, "temperature": null, "top_p": null, "max_output_tokens": 65536, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Google", "harness": null }, "minimax/MiniMax-M3.1-flash": { "accuracy": 91.414, "latency": 1357.85, "stderr": 1.628, "cost_per_test": 0.463122, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "MiniMax", "harness": null }, "tencent/hy4-preview": { "accuracy": 91.414, "latency": 1697.31, "stderr": 1.628, "cost_per_test": 0.869928, "temperature": 0.9, "top_p": 1.0, "max_output_tokens": 64000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Tencent", "harness": null }, "kimi/kimi-k3": { "accuracy": 91.246, "latency": 2825.84, "stderr": 1.643, "cost_per_test": 5.839684, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 262144, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Moonshot AI", "harness": null }, "xiaomi/mimo-v2.6-flash": { "accuracy": 90.572, "latency": 917.02, "stderr": 1.698, "cost_per_test": 0.07138, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "xiaomi/mimo-v2.6-pro": { "accuracy": 90.067, "latency": 1712.104, "stderr": 1.738, "cost_per_test": 0.16563, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Xiaomi", "harness": null }, "deepseek/deepseek-v4.1-flash-1017": { "accuracy": 89.899, "latency": 1431.654, "stderr": 1.752, "cost_per_test": 0.688859, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "inception/mercury-3": { "accuracy": 89.394, "latency": 714.676, "stderr": 1.79, "cost_per_test": 0.134735, "temperature": null, "top_p": null, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "high", "verbosity": null, "compute_effort": null, "provider": "Inception", "harness": null }, "anthropic/claude-sonnet-5-5": { "accuracy": 89.226, "latency": 4937.706, "stderr": 1.802, "cost_per_test": 17.206836, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "openai/gpt-6-astra": { "accuracy": 89.057, "latency": 1732.526, "stderr": 1.814, "cost_per_test": 17.857837, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reflection/Beam-1.0": { "accuracy": 88.889, "latency": 109.104, "stderr": 1.827, "cost_per_test": null, "temperature": 0.7, "top_p": 0.99, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reflection", "harness": null }, "deepseek/deepseek-v4-pro-0813": { "accuracy": 88.889, "latency": 477.316, "stderr": 1.827, "cost_per_test": 0.622985, "temperature": null, "top_p": null, "max_output_tokens": 384000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "DeepSeek", "harness": null }, "zai/glm-5.3-flash": { "accuracy": 88.552, "latency": 694.844, "stderr": 1.851, "cost_per_test": 0.286703, "temperature": 1.0, "top_p": 0.95, "max_output_tokens": 131072, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "Fireworks AI", "harness": null }, "openai/gpt-6.1-sol": { "accuracy": 87.205, "latency": 2123.426, "stderr": 1.942, "cost_per_test": 3.053435, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-6-luna": { "accuracy": 85.859, "latency": 1480.506, "stderr": 2.025, "cost_per_test": 0.268224, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": "max", "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "anthropic/claude-haiku-5-5": { "accuracy": 84.343, "latency": 3043.182, "stderr": 2.112, "cost_per_test": 5.01937, "temperature": null, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "anthropic/claude-opus-5-5": { "accuracy": 83.333, "latency": 3518.708, "stderr": 2.166, "cost_per_test": 24.223778, "temperature": 1.0, "top_p": null, "max_output_tokens": 128000, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": "max", "provider": "Anthropic", "harness": null }, "thinkingmachines/inkling": { "accuracy": 77.946, "latency": 374.084, "stderr": 2.41, "cost_per_test": 1.451903, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 256000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null }, "thinkingmachines/inkling-small": { "accuracy": 71.886, "latency": 433.434, "stderr": 2.613, "cost_per_test": 0.180201, "temperature": 1.0, "top_p": 1.0, "max_output_tokens": 262000, "reasoning": null, "reasoning_effort": "0.99", "verbosity": null, "compute_effort": null, "provider": "Thinkingmachines", "harness": null } } } }